From 3197ac70a8d56c206e3b981702780912249335f9 Mon Sep 17 00:00:00 2001 From: isiamionau Date: Fri, 22 Jul 2022 14:11:09 -0500 Subject: [PATCH 1/7] kafka producer settings were added --- attributes/default.rb | 7 + recipes/kafka_producer.rb | 43 ++++ .../org.apache.karaf.features.cfg.erb | 206 ++++++++++++++++++ ...rg.opennms.features.kafka.producer.cfg.erb | 3 + 4 files changed, 259 insertions(+) create mode 100644 recipes/kafka_producer.rb create mode 100644 templates/default/horizon-26/org.apache.karaf.features.cfg.erb create mode 100644 templates/default/horizon-26/org.opennms.features.kafka.producer.cfg.erb diff --git a/attributes/default.rb b/attributes/default.rb index eb825724..f22a2034 100644 --- a/attributes/default.rb +++ b/attributes/default.rb @@ -2123,3 +2123,10 @@ default['opennms']['bin']['cookbook'] = 'opennms' default['opennms']['bin']['return_code'] = false + +#Kafka +default['opennms']['karaf']['addl_features'] = [] +default['opennms']['kafka']['producer'] = {'eventTopic' => 'events', 'alarmTopic' => 'alarms', 'alarmFeedbackTopic' => 'alarmFeedback', 'nodeTopic' => 'nodes', 'topologyVertexTopic' => 'vertices', + 'topologyEdgeTopic' => 'edges', 'metricTopic' => 'metrics', 'eventFilter' => '', 'alarmFilter' => '', 'forward.metrics' => false, + 'nodeRefreshTimeoutMs' => 300000, 'alarmSyncIntervalMs' => 300000, 'suppressIncrementalAlarms' => true, 'kafkaSendQueueCapacity' => 1000, + 'startAlarmSyncWithCleanState' => false} diff --git a/recipes/kafka_producer.rb b/recipes/kafka_producer.rb new file mode 100644 index 00000000..a073d9d1 --- /dev/null +++ b/recipes/kafka_producer.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true +# +# Cookbook Name:: opennms-cookbook +# Recipe:: kafka_producer +# +# Copyright 2015-2022, ConvergeOne +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +mv = Opennms::Helpers.major(node['opennms']['version']) +template_dir = if node['opennms']['conf']['cookbook'] == 'opennms' + "horizon-#{format(mv, version: node['opennms']['version'])}/" + else + '' + end + +template "#{node['opennms']['conf']['home']}/etc/org.opennms.features.kafka.producer.cfg" do + source "#{template_dir}org.opennms.features.kafka.producer.cfg.erb" + owner 'root' + group 'root' + mode 00664 + only_if { mv.to_i >= 26} + notifies :restart, 'service[opennms]', :delayed +end + +template "#{node['opennms']['conf']['home']}/etc/org.apache.karaf.features.cfg" do + source "#{template_dir}org.apache.karaf.features.cfg.erb" + owner 'root' + group 'root' + mode 00664 + only_if { mv.to_i >= 26} + notifies :restart, 'service[opennms]', :delayed +end diff --git a/templates/default/horizon-26/org.apache.karaf.features.cfg.erb b/templates/default/horizon-26/org.apache.karaf.features.cfg.erb new file mode 100644 index 00000000..65745340 --- /dev/null +++ b/templates/default/horizon-26/org.apache.karaf.features.cfg.erb @@ -0,0 +1,206 @@ +################################################################################ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +# +# Comma separated list of features repositories to register by default +# +#featuresRepositories = \ +# mvn:org.apache.karaf.features/spring/4.2.6/xml/features, \ +# mvn:org.apache.karaf.features/standard/4.2.6/xml/features, \ +# mvn:org.apache.karaf.features/framework/4.2.6/xml/features, \ +# mvn:org.apache.karaf.features/enterprise/4.2.6/xml/features + +# OPENNMS: Override Karaf's standard-4.2.6-features.xml with our customized version with the HTTP bridge code +# and override Karaf's spring-4.2.6-features.xml and spring-legacy-4.2.6-features.xml with our versions +# that point to the modified standard features. +# OPENNMS: Include the main product features file generated by the mvn:org.opennms.karaf/opennms project +featuresRepositories = \ + mvn:org.opennms.karaf/opennms/26.2.2/xml/standard, \ + mvn:org.opennms.karaf/opennms/26.2.2/xml/spring, \ + mvn:org.opennms.karaf/opennms/26.2.2/xml/spring-legacy, \ + mvn:org.apache.karaf.features/framework/4.2.6/xml/features, \ + mvn:org.opennms.karaf/opennms/26.2.2/xml/features, \ + mvn:org.opennms.karaf/opennms/26.2.2/xml/karaf-extensions +# +# Comma separated list of features to install at startup +# +#featuresBoot = \ +# instance, \ +# package, \ +# log, \ +# ssh, \ +# framework, \ +# system, \ +# eventadmin, \ +# feature, \ +# shell, \ +# management, \ +# service, \ +# jaas, \ +# deployer, \ +# diagnostic, \ +# wrap, \ +# bundle, \ +# config, \ +# kar + +# OPENNMS: Parenthetize standard Karaf boot features, add OpenNMS product features +featuresBoot = ( \ + instance, \ + package, \ + log, \ + ssh, \ + aries-blueprint, \ + framework, \ + system, \ + eventadmin, \ + feature, \ + shell, \ + management, \ + service, \ + jaas, \ + shell-compat, \ + deployer, \ + diagnostic, \ + wrap, \ + bundle, \ + config, \ + kar \ + ), \ + http,\ + http-whiteboard,\ + opennms-jaas-login-module,\ + karaf-extender, \ + opennms-osgi-core-rest, \ + opennms-health-rest,\ + datachoices, \ + opennms-activemq-shell, \ + opennms-collection-commands, \ + opennms-core-ipc-rpc-commands, \ + opennms-core-ipc-kafka-shell, \ + opennms-dhcpd, \ + opennms-events-commands, \ + opennms-send-event-command, \ + opennms-identity, \ + opennms-icmp-commands, \ + opennms-snmp-commands, \ + opennms-telemetry-collection,\ + opennms-telemetry-bmp,\ + opennms-telemetry-jti,\ + opennms-telemetry-nxos,\ + opennms-telemetry-graphite,\ + opennms-telemetry-openconfig, \ + opennms-dnsresolver-shell,\ + opennms-dnsresolver-netty,\ + opennms-flows,\ + opennms-topology-runtime-browsers,\ + opennms-topology-runtime-linkd,\ + opennms-topology-runtime-vmware,\ + opennms-topology-runtime-application,\ + opennms-topology-runtime-bsm,\ + opennms-topology-runtime-pathoutage,\ + opennms-provisioning-shell,\ + opennms-poller-shell,\ + opennms-topology-runtime-graphml,\ + opennms-topology-runtime-asset,\ + opennms-api-layer,\ + opennms-blobstore-postgres,\ + opennms-jsonstore-shell,\ + opennms-threshold-states-shell,\ + opennms-measurements-shell,\ + opennms-graphml,\ + opennms-graphs,\ + osgi-nrtg-local,\ + vaadin-node-maps,\ + vaadin-snmp-events-and-metrics, \ + vaadin-dashboard, \ + dashlet-summary, \ + dashlet-alarms, \ + dashlet-bsm, \ + dashlet-map, \ + dashlet-image, \ + dashlet-charts, \ + dashlet-grafana, \ + dashlet-rtc, \ + dashlet-rrd, \ + dashlet-ksc, \ + dashlet-topology, \ + dashlet-url, \ + dashlet-surveillance, \ + vaadin-surveillance-views, \ + vaadin-jmxconfiggenerator, \ + vaadin-adminpage, \ + org.opennms.features.bsm.shell-commands, \ + geolocation, \ + ifttt-integration, \ + opennms-endpoints-grafana, \ + opennms-reporting, \ + <% node['opennms']['karaf']['addl_features'].each do |v| -%> + <%=v%> + <% end -%> + opennms-karaf-health + +# Ensure that the 'opennms-karaf-health' feature is installed *last* + +# +# Resource repositories (OBR) that the features resolver can use +# to resolve requirements/capabilities +# +# The format of the resourceRepositories is +# resourceRepositories=[xml:url|json:url],... +# for Instance: +# +#resourceRepositories=xml:http://host/path/to/index.xml +# or +#resourceRepositories=json:http://host/path/to/index.json +# + +# +# Defines if the boot features are started in asynchronous mode (in a dedicated thread) +# +featuresBootAsynchronous=false + +# +# Service requirements enforcement +# +# By default, the feature resolver checks the service requirements/capabilities of +# bundles for new features (xml schema >= 1.3.0) in order to automatically installs +# the required bundles. +# The following flag can have those values: +# - disable: service requirements are completely ignored +# - default: service requirements are ignored for old features +# - enforce: service requirements are always verified +# +# OPENNMS: We need to disable this so that Pax Web doesn't complain about bring run +# inside our Felix bridge HTTP service (which does not implement the Pax Web services) +serviceRequirements=disable + +# +# Store cfg file for config element in feature +# +#configCfgStore=true + +# +# Configuration of features processing mechanism (overrides, blacklisting, modification of features) +# XML file defines instructions related to features processing +# versions.properties may declare properties to resolve placeholders in XML file +# both files are relative to ${karaf.etc} +# +#featureProcessing=org.apache.karaf.features.xml +#featureProcessingVersions=versions.properties diff --git a/templates/default/horizon-26/org.opennms.features.kafka.producer.cfg.erb b/templates/default/horizon-26/org.opennms.features.kafka.producer.cfg.erb new file mode 100644 index 00000000..10d62123 --- /dev/null +++ b/templates/default/horizon-26/org.opennms.features.kafka.producer.cfg.erb @@ -0,0 +1,3 @@ +<% node['opennms']['kafka']['producer'].each do |k,v| -%> +<%=k%>=<%=v%> +<% end -%> From d4d055a84bae3d4ef841cb787ff55f2c514eb723 Mon Sep 17 00:00:00 2001 From: isiamionau Date: Sat, 23 Jul 2022 06:41:07 -0500 Subject: [PATCH 2/7] kafka producer settings were added --- templates/default/horizon-26/org.apache.karaf.features.cfg.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/default/horizon-26/org.apache.karaf.features.cfg.erb b/templates/default/horizon-26/org.apache.karaf.features.cfg.erb index 65745340..9106f9cd 100644 --- a/templates/default/horizon-26/org.apache.karaf.features.cfg.erb +++ b/templates/default/horizon-26/org.apache.karaf.features.cfg.erb @@ -152,7 +152,7 @@ featuresBoot = ( \ opennms-endpoints-grafana, \ opennms-reporting, \ <% node['opennms']['karaf']['addl_features'].each do |v| -%> - <%=v%> + <%=v%>, \ <% end -%> opennms-karaf-health From 7180c8a5a611bf0bb88c6503055c165187a88bdd Mon Sep 17 00:00:00 2001 From: isiamionau Date: Sat, 23 Jul 2022 14:42:38 -0500 Subject: [PATCH 3/7] kafka producer settings were added --- attributes/default.rb | 1 + recipes/kafka_producer.rb | 9 +++++++++ .../org.opennms.features.kafka.producer.client.cfg.erb | 3 +++ 3 files changed, 13 insertions(+) create mode 100644 templates/default/horizon-26/org.opennms.features.kafka.producer.client.cfg.erb diff --git a/attributes/default.rb b/attributes/default.rb index f22a2034..d51fe46d 100644 --- a/attributes/default.rb +++ b/attributes/default.rb @@ -2130,3 +2130,4 @@ 'topologyEdgeTopic' => 'edges', 'metricTopic' => 'metrics', 'eventFilter' => '', 'alarmFilter' => '', 'forward.metrics' => false, 'nodeRefreshTimeoutMs' => 300000, 'alarmSyncIntervalMs' => 300000, 'suppressIncrementalAlarms' => true, 'kafkaSendQueueCapacity' => 1000, 'startAlarmSyncWithCleanState' => false} +default['opennms']['kafka']['producer'] ['client'] = {} diff --git a/recipes/kafka_producer.rb b/recipes/kafka_producer.rb index a073d9d1..ce65f0a2 100644 --- a/recipes/kafka_producer.rb +++ b/recipes/kafka_producer.rb @@ -33,6 +33,15 @@ notifies :restart, 'service[opennms]', :delayed end +template "#{node['opennms']['conf']['home']}/etc/org.opennms.features.kafka.producer.client.cfg" do + source "#{template_dir}org.opennms.features.kafka.producer.client.cfg.erb" + owner 'root' + group 'root' + mode 00664 + only_if { mv.to_i >= 26} + notifies :restart, 'service[opennms]', :delayed +end + template "#{node['opennms']['conf']['home']}/etc/org.apache.karaf.features.cfg" do source "#{template_dir}org.apache.karaf.features.cfg.erb" owner 'root' diff --git a/templates/default/horizon-26/org.opennms.features.kafka.producer.client.cfg.erb b/templates/default/horizon-26/org.opennms.features.kafka.producer.client.cfg.erb new file mode 100644 index 00000000..c791649c --- /dev/null +++ b/templates/default/horizon-26/org.opennms.features.kafka.producer.client.cfg.erb @@ -0,0 +1,3 @@ +<% node['opennms']['kafka']['producer']['client'].each do |k,v| -%> +<%=k%>=<%=v%> +<% end -%> From 39ec66b66f16f260468da554ab6a5df83458c276 Mon Sep 17 00:00:00 2001 From: isiamionau Date: Mon, 25 Jul 2022 12:06:09 -0500 Subject: [PATCH 4/7] refactoring --- attributes/default.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/attributes/default.rb b/attributes/default.rb index d51fe46d..c5ba3273 100644 --- a/attributes/default.rb +++ b/attributes/default.rb @@ -2130,4 +2130,4 @@ 'topologyEdgeTopic' => 'edges', 'metricTopic' => 'metrics', 'eventFilter' => '', 'alarmFilter' => '', 'forward.metrics' => false, 'nodeRefreshTimeoutMs' => 300000, 'alarmSyncIntervalMs' => 300000, 'suppressIncrementalAlarms' => true, 'kafkaSendQueueCapacity' => 1000, 'startAlarmSyncWithCleanState' => false} -default['opennms']['kafka']['producer'] ['client'] = {} +default['opennms']['kafka']['producer']['client'] = {'bootstrap.servers' => '127.0.0.1:9092'} From d80f654ed86d402d758a5fc94abdc0aaedf10748 Mon Sep 17 00:00:00 2001 From: isiamionau Date: Mon, 25 Jul 2022 15:04:38 -0500 Subject: [PATCH 5/7] refactoring --- attributes/default.rb | 2 +- metadata.rb | 2 +- recipes/kafka_producer.rb | 6 +++--- .../org.opennms.features.kafka.producer.client.cfg.erb | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/attributes/default.rb b/attributes/default.rb index c5ba3273..3a926201 100644 --- a/attributes/default.rb +++ b/attributes/default.rb @@ -2130,4 +2130,4 @@ 'topologyEdgeTopic' => 'edges', 'metricTopic' => 'metrics', 'eventFilter' => '', 'alarmFilter' => '', 'forward.metrics' => false, 'nodeRefreshTimeoutMs' => 300000, 'alarmSyncIntervalMs' => 300000, 'suppressIncrementalAlarms' => true, 'kafkaSendQueueCapacity' => 1000, 'startAlarmSyncWithCleanState' => false} -default['opennms']['kafka']['producer']['client'] = {'bootstrap.servers' => '127.0.0.1:9092'} +default['opennms']['kafka']['producer_client'] = {'bootstrap.servers' => '127.0.0.1:9092'} diff --git a/metadata.rb b/metadata.rb index 5ae88b66..c9c0f55c 100644 --- a/metadata.rb +++ b/metadata.rb @@ -9,7 +9,7 @@ source_url 'https://github.com/dschlenk/opennms-cookbook' supports 'centos', '>= 6.0' supports 'redhat', '>= 6.0' -version '28.0.0' +version '28.1.0' depends 'hostsfile' depends 'build-essential' depends 'postgresql' diff --git a/recipes/kafka_producer.rb b/recipes/kafka_producer.rb index ce65f0a2..982e4817 100644 --- a/recipes/kafka_producer.rb +++ b/recipes/kafka_producer.rb @@ -29,7 +29,7 @@ owner 'root' group 'root' mode 00664 - only_if { mv.to_i >= 26} + only_if { mv.to_i == 26} notifies :restart, 'service[opennms]', :delayed end @@ -38,7 +38,7 @@ owner 'root' group 'root' mode 00664 - only_if { mv.to_i >= 26} + only_if { mv.to_i == 26} notifies :restart, 'service[opennms]', :delayed end @@ -47,6 +47,6 @@ owner 'root' group 'root' mode 00664 - only_if { mv.to_i >= 26} + only_if { mv.to_i == 26} notifies :restart, 'service[opennms]', :delayed end diff --git a/templates/default/horizon-26/org.opennms.features.kafka.producer.client.cfg.erb b/templates/default/horizon-26/org.opennms.features.kafka.producer.client.cfg.erb index c791649c..08d77f01 100644 --- a/templates/default/horizon-26/org.opennms.features.kafka.producer.client.cfg.erb +++ b/templates/default/horizon-26/org.opennms.features.kafka.producer.client.cfg.erb @@ -1,3 +1,3 @@ -<% node['opennms']['kafka']['producer']['client'].each do |k,v| -%> +<% node['opennms']['kafka']['producer_client'].each do |k,v| -%> <%=k%>=<%=v%> <% end -%> From c762cac335c436dfac29d7ba7e28f65585b79c84 Mon Sep 17 00:00:00 2001 From: isiamionau Date: Mon, 25 Jul 2022 17:02:28 -0500 Subject: [PATCH 6/7] some tests for kafka producer --- test/integration/kafka/kafka_settings.rb | 25 ++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 test/integration/kafka/kafka_settings.rb diff --git a/test/integration/kafka/kafka_settings.rb b/test/integration/kafka/kafka_settings.rb new file mode 100644 index 00000000..54c4b602 --- /dev/null +++ b/test/integration/kafka/kafka_settings.rb @@ -0,0 +1,25 @@ + describe file('/opt/opennms/etc/org.opennms.features.kafka.producer.client.cfg') do + its('content') { should include 'bootstrap.servers=127.0.0.1:9092' } + end + + describe file('/opt/opennms/etc/org.opennms.features.kafka.producer.cfg') do + its('content') { should include 'eventTopic=events' } + its('content') { should include 'alarmTopic=alarms' } + its('content') { should include 'alarmFeedbackTopic=alarmFeedback' } + its('content') { should include 'nodeTopic=nodes' } + its('content') { should include 'topologyVertexTopic=vertices' } + its('content') { should include 'topologyEdgeTopic=edges' } + its('content') { should include 'metricTopic=metrics' } + its('content') { should include 'eventFilter=' } + its('content') { should include 'alarmFilter=' } + its('content') { should include 'forward.metrics=false' } + its('content') { should include 'nodeRefreshTimeoutMs=300000' } + its('content') { should include 'alarmSyncIntervalMs=300000' } + its('content') { should include 'suppressIncrementalAlarms=true' } + its('content') { should include 'kafkaSendQueueCapacity=1000' } + its('content') { should include 'startAlarmSyncWithCleanState=false' } + end + + describe file('/opt/opennms/etc/org.apache.karaf.features.cfg') do + its('content') { should include 'opennms-kafka-producer' } + end From 5eac985ff5318d9cec4e06a17c9b571454eea3da Mon Sep 17 00:00:00 2001 From: isiamionau Date: Tue, 26 Jul 2022 08:52:14 -0500 Subject: [PATCH 7/7] some tests for kafka producer --- .kitchen.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.kitchen.yml b/.kitchen.yml index 2b76a46e..3281659c 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -29644,9 +29644,12 @@ suites: - recipe[oracle_java8::default] - recipe[opennms::default] - recipe[onms_lwrp_test::webopts] + - recipe[opennms::kafka_producer] attributes: opennms: version: 26.2.2-1 + karaf: + addl_features: ['opennms-kafka-producer' ] verifier: inspec_tests: - path: test/integration/default