From 33fe442c4307f30f45fd2564f6fcc2989ae9d83f Mon Sep 17 00:00:00 2001 From: Hugo Haakseth Date: Thu, 11 Apr 2024 07:42:32 +0200 Subject: [PATCH 1/7] Fix replset acceptance testing --- .../provider/mongodb_database/mongodb.rb | 14 +- spec/acceptance/replset_spec.rb | 238 +++++++++++++++--- .../provider/mongodb_database/mongodb_spec.rb | 2 +- 3 files changed, 216 insertions(+), 38 deletions(-) diff --git a/lib/puppet/provider/mongodb_database/mongodb.rb b/lib/puppet/provider/mongodb_database/mongodb.rb index 1cf061d25..7b91895a7 100644 --- a/lib/puppet/provider/mongodb_database/mongodb.rb +++ b/lib/puppet/provider/mongodb_database/mongodb.rb @@ -9,12 +9,16 @@ def self.instances require 'json' - pre_cmd = 'db.getMongo().setReadPref("primaryPreferred")' - dbs = JSON.parse mongo_eval("#{pre_cmd};EJSON.stringify(db.getMongo().getDBs())") + if db_ismaster + dbs = JSON.parse mongo_eval('EJSON.stringify(db.getMongo().getDBs())') - dbs['databases'].map do |db| - new(name: db['name'], - ensure: :present) + dbs['databases'].map do |db| + new(name: db['name'], + ensure: :present) + end + else + Puppet.warning 'Database info is available only from master host' + [] end end diff --git a/spec/acceptance/replset_spec.rb b/spec/acceptance/replset_spec.rb index 68d0a3c37..4393ed56c 100644 --- a/spec/acceptance/replset_spec.rb +++ b/spec/acceptance/replset_spec.rb @@ -8,9 +8,6 @@ if hosts.length > 1 && supported_version?(default[:platform], repo_version) describe 'mongodb_replset resource' do after :all do - # Have to drop the DB to disable replsets for further testing - on hosts, %{mongosh local --verbose --eval 'db.dropDatabase()'} - pp = <<-EOS class { 'mongodb::globals': #{repo_ver_param} @@ -20,10 +17,8 @@ class { 'mongodb::globals': package_ensure => purged, service_ensure => stopped } - if $::osfamily == 'RedHat' { - class { 'mongodb::client': - ensure => purged - } + -> class { 'mongodb::client': + ensure => purged } EOS @@ -36,12 +31,10 @@ class { 'mongodb::globals': #{repo_ver_param} } -> class { 'mongodb::server': - bind_ip => '0.0.0.0', + bind_ip => ['0.0.0.0'], replset => 'test', } - if $::osfamily == 'RedHat' { - class { 'mongodb::client': } - } + -> class { 'mongodb::client': } EOS apply_manifest_on(hosts.reverse, pp, catch_failures: true) @@ -59,11 +52,81 @@ class { 'mongodb::client': } expect(r.stdout).to match %r{#{hosts[0]}:27017} expect(r.stdout).to match %r{#{hosts[1]}:27017} end + sleep(30) + on(hosts_as('slave'), 'mongosh --quiet --eval "EJSON.stringify(rs.conf())"') do |r| + expect(r.stdout).to match %r{#{hosts[0]}:27017} + expect(r.stdout).to match %r{#{hosts[1]}:27017} + end end it 'inserts data on the master' do + on hosts_as('master'), %{mongosh --verbose --eval 'db.test.insertOne({name:"test1",value:"some value"})'} + end + + it 'checks the data on the master' do + on hosts_as('master'), %{mongosh --verbose --eval 'EJSON.stringify(db.test.findOne({name:"test1"}))'} do |r| + expect(r.stdout).to match %r{some value} + end + end + + it 'checks the data on the slave' do + sleep(10) + on hosts_as('slave'), %{mongosh --verbose --eval 'EJSON.stringify(db.test.findOne({name:"test1"}))'} do |r| + expect(r.stdout).to match %r{some value} + end + end + end + + describe 'mongodb::server with replset_members' do + after :all do + pp = <<-EOS + class { 'mongodb::globals': + #{repo_ver_param} + } + -> class { 'mongodb::server': + ensure => absent, + package_ensure => purged, + service_ensure => stopped + } + -> class { 'mongodb::client': + ensure => purged + } + EOS + + apply_manifest_on(hosts.reverse, pp, catch_failures: true) + end + + it 'configures mongo on both nodes' do + pp = <<-EOS + class { 'mongodb::globals': + #{repo_ver_param} + } + -> class { 'mongodb::server': + bind_ip => ['0.0.0.0'], + replset => 'test', + replset_members => [#{hosts.map { |x| "'#{x}:27017'" }.join(',')}], + } + -> class { 'mongodb::client': } + EOS + + apply_manifest_on(hosts, pp, catch_failures: true) + apply_manifest_on(hosts, pp, catch_changes: true) + end + + it 'sets up the replset with puppet' do + on(hosts_as('master'), 'mongosh --quiet --eval "EJSON.stringify(rs.conf())"') do |r| + expect(r.stdout).to match %r{#{hosts[0]}:27017} + expect(r.stdout).to match %r{#{hosts[1]}:27017} + end sleep(30) - on hosts_as('master'), %{mongosh --verbose --eval 'db.test.save({name:"test1",value:"some value"})'} + on(hosts_as('slave'), 'mongosh --quiet --eval "EJSON.stringify(rs.conf())"') do |r| + expect(r.stdout).to match %r{#{hosts[0]}:27017} + expect(r.stdout).to match %r{#{hosts[1]}:27017} + end + end + + it 'inserts data on the master' do + on hosts_as('master'), %{mongosh --verbose --eval 'db.getMongo().setReadPref("primaryPreferred");db.test.insertOne({name:"test1",value:"some value"})'} end it 'checks the data on the master' do @@ -74,7 +137,7 @@ class { 'mongodb::client': } it 'checks the data on the slave' do sleep(10) - on hosts_as('slave'), %{mongosh --verbose --eval 'db.getMongo().setReadPref("primaryPreferred"); EJSON.stringify(db.test.findOne({name:"test1"}))'} do |r| + on hosts_as('slave'), %{mongosh --verbose --eval 'EJSON.stringify(db.test.findOne({name:"test1"}))'} do |r| expect(r.stdout).to match %r{some value} end end @@ -82,9 +145,6 @@ class { 'mongodb::client': } describe 'mongodb_replset resource with auth => true' do after :all do - # Have to drop the DB to disable replsets for further testing - on hosts, %{mongosh local --verbose --eval 'db.dropDatabase()'} - pp = <<-EOS class { 'mongodb::globals': #{repo_ver_param} @@ -94,28 +154,36 @@ class { 'mongodb::globals': package_ensure => purged, service_ensure => stopped } - if $::osfamily == 'RedHat' { - class { 'mongodb::client': - ensure => purged - } + -> class { 'mongodb::client': + ensure => purged } EOS apply_manifest_on(hosts.reverse, pp, catch_failures: true) end + let(:keyfile_path) do + os_family = fact('os.family') + if os_family == 'RedHat' + '/var/lib/mongo' + else + '/var/lib/mongodb' + end + end + it 'configures mongo on both nodes' do pp = <<~EOS class { 'mongodb::globals': #{repo_ver_param} - } -> - class { 'mongodb::server': + } + -> class { 'mongodb::server': admin_username => 'admin', admin_password => 'password', auth => true, - bind_ip => '0.0.0.0', + store_creds => true, + bind_ip => ['0.0.0.0'], replset => 'test', - keyfile => '/var/lib/mongodb/mongodb-keyfile', + keyfile => "#{keyfile_path}/mongodb-keyfile", key => '+dxlTrury7xtD0FRqFf3YWGnKqWAtlyauuemxuYuyz9POPUuX1Uj3chGU8MFMHa7 UxASqex7NLMALQXHL+Th4T3dyb6kMZD7KiMcJObO4M+JLiX9drcTiifsDEgGMi7G vYn3pWSm5TTDrHJw7RNWfMHw3sHk0muGQcO+0dWv3sDJ6SiU8yOKRtYcTEA15GbP @@ -131,11 +199,9 @@ class { 'mongodb::server': g+Bybk5qHv1b7M8Tv9/I/BRXcpLHeIkMICMY8sVPGmP8xzL1L3i0cws8p5h0zPBa YG/QX0BmltAni8owgymFuyJgvr/gaRX4WHbKFD+9nKpqJ3ocuVNuCDsxDqLsJEME nc1ohyB0lNt8lHf1U00mtgDSV3fwo5LkwhRi6d+bDBTL/C6MZETMLdyCqDlTdUWG - YXIsJ0gYcu9XG3mx10LbdPJvxSMg' - } - if $::osfamily == 'RedHat' { - include mongodb::client + YXIsJ0gYcu9XG3mx10LbdPJvxSMg', } + -> class { 'mongodb::client': } EOS apply_manifest_on(hosts.reverse, pp, catch_failures: true) @@ -145,9 +211,16 @@ class { 'mongodb::server': it 'sets up the replset with puppet' do pp = <<~EOS mongodb_replset { 'test': - auth_enabled => true, - members => [#{hosts.map { |x| "'#{x}:27017'" }.join(',')}], - before => Mongodb_user['admin'] + members => [#{hosts.map { |x| "'#{x}:27017'" }.join(',')}], + } + -> mongodb_user { "User admin on db admin": + ensure => present, + password_hash => mongodb_password('admin', 'password'), + username => 'admin', + database => 'admin', + roles => ['userAdmin', 'readWrite', 'dbAdmin', 'dbAdminAnyDatabase', 'readAnyDatabase', + 'readWriteAnyDatabase', 'userAdminAnyDatabase', 'clusterAdmin', + 'clusterManager', 'clusterMonitor', 'hostManager', 'root', 'restore',], } EOS apply_manifest_on(hosts_as('master'), pp, catch_failures: true) @@ -156,11 +229,112 @@ class { 'mongodb::server': expect(r.stdout).to match %r{#{hosts[0]}:27017} expect(r.stdout).to match %r{#{hosts[1]}:27017} end + sleep(30) + on(hosts_as('slave'), 'mongosh --quiet --eval "load(\'/root/.mongoshrc.js\');EJSON.stringify(rs.conf())"') do |r| + expect(r.stdout).to match %r{#{hosts[0]}:27017} + expect(r.stdout).to match %r{#{hosts[1]}:27017} + end end it 'inserts data on the master' do + on hosts_as('master'), %{mongosh test --verbose --eval 'load("/root/.mongoshrc.js");db.dummyData.insertOne({"created_by_puppet": 1})'} + end + + it 'checks the data on the master' do + on hosts_as('master'), %{mongosh test --verbose --eval 'load("/root/.mongoshrc.js");EJSON.stringify(db.dummyData.findOne())'} do |r| + expect(r.stdout).to match %r{created_by_puppet} + end + end + + it 'checks the data on the slave' do + sleep(10) + on hosts_as('slave'), %{mongosh test --verbose --eval 'load("/root/.mongoshrc.js");db.getMongo().setReadPref("primaryPreferred");EJSON.stringify(db.dummyData.findOne())'} do |r| + expect(r.stdout).to match %r{created_by_puppet} + end + end + end + + describe 'mongodb::server with replset_members and auth => true' do + after :all do + pp = <<-EOS + class { 'mongodb::globals': + #{repo_ver_param} + } + -> class { 'mongodb::server': + ensure => absent, + package_ensure => purged, + service_ensure => stopped + } + -> class { 'mongodb::client': + ensure => purged + } + EOS + + apply_manifest_on(hosts.reverse, pp, catch_failures: true) + end + + let(:keyfile_path) do + os_family = fact('os.family') + if os_family == 'RedHat' + '/var/lib/mongo' + else + '/var/lib/mongodb' + end + end + + it 'configures mongo on both nodes' do + pp = <<~EOS + class { 'mongodb::globals': + #{repo_ver_param} + } + -> class { 'mongodb::client': } + -> class { 'mongodb::server': + admin_username => 'admin', + admin_password => 'password', + auth => true, + store_creds => true, + create_admin => true, + bind_ip => ['0.0.0.0'], + replset => 'test', + replset_members => [#{hosts.map { |x| "'#{x}:27017'" }.join(',')}], + keyfile => "#{keyfile_path}/mongodb-keyfile", + key => '+dxlTrury7xtD0FRqFf3YWGnKqWAtlyauuemxuYuyz9POPUuX1Uj3chGU8MFMHa7 + UxASqex7NLMALQXHL+Th4T3dyb6kMZD7KiMcJObO4M+JLiX9drcTiifsDEgGMi7G + vYn3pWSm5TTDrHJw7RNWfMHw3sHk0muGQcO+0dWv3sDJ6SiU8yOKRtYcTEA15GbP + ReDZuHFy1T1qhk5NIt6pTtPGsZKSm2wAWIOa2f2IXvpeQHhjxP8aDFb3fQaCAqOD + R7hrimqq0Nickfe8RLA89iPXyadr/YeNBB7w7rySatQBzwIbBUVGNNA5cxCkwyx9 + E5of3xi7GL9xNxhQ8l0JEpofd4H0y0TOfFDIEjc7cOnYlKAHzBgog4OcFSILgUaF + kHuTMtv0pj+MMkW2HkeXETNII9XE1+JiZgHY08G7yFEJy87ttUoeKmpbI6spFz5U + 4K0amj+N6SOwXaS8uwp6kCqay0ERJLnw+7dKNKZIZdGCrrBxcZ7wmR/wLYrxvHhZ + QpeXTxgD5ebwCR0cf3Xnb5ql5G/HHKZDq8LTFHwELNh23URGPY7K7uK+IF6jSEhq + V2H3HnWV9teuuJ5he9BB/pLnyfjft6KUUqE9HbaGlX0f3YBk/0T3S2ESN4jnfRTQ + ysAKvQ6NasXkzqXktu8X4fS5QNqrFyqKBZSWxttfJBKXnT0TxamCKLRx4AgQglYo + 3KRoyfxXx6G+AjP1frDJxFAFEIgEFqRk/FFuT/y9LpU+3cXYX1Gt6wEatgmnBM3K + g+Bybk5qHv1b7M8Tv9/I/BRXcpLHeIkMICMY8sVPGmP8xzL1L3i0cws8p5h0zPBa + YG/QX0BmltAni8owgymFuyJgvr/gaRX4WHbKFD+9nKpqJ3ocuVNuCDsxDqLsJEME + nc1ohyB0lNt8lHf1U00mtgDSV3fwo5LkwhRi6d+bDBTL/C6MZETMLdyCqDlTdUWG + YXIsJ0gYcu9XG3mx10LbdPJvxSMg', + } + EOS + + apply_manifest_on(hosts, pp, catch_failures: true) + apply_manifest_on(hosts, pp, catch_changes: true) + end + + it 'sets up the replset with puppet' do + on(hosts_as('master'), 'mongosh --quiet --eval "load(\'/root/.mongoshrc.js\');EJSON.stringify(rs.conf())"') do |r| + expect(r.stdout).to match %r{#{hosts[0]}:27017} + expect(r.stdout).to match %r{#{hosts[1]}:27017} + end sleep(30) - on hosts_as('master'), %{mongosh test --verbose --eval 'load("/root/.mongoshrc.js");db.dummyData.insert({"created_by_puppet": 1})'} + on(hosts_as('slave'), 'mongosh --quiet --eval "load(\'/root/.mongoshrc.js\');EJSON.stringify(rs.conf())"') do |r| + expect(r.stdout).to match %r{#{hosts[0]}:27017} + expect(r.stdout).to match %r{#{hosts[1]}:27017} + end + end + + it 'inserts data on the master' do + on hosts_as('master'), %{mongosh test --verbose --eval 'load("/root/.mongoshrc.js");db.dummyData.insertOne({"created_by_puppet": 1})'} end it 'checks the data on the master' do diff --git a/spec/unit/puppet/provider/mongodb_database/mongodb_spec.rb b/spec/unit/puppet/provider/mongodb_database/mongodb_spec.rb index 8f25fb384..c0a4825de 100644 --- a/spec/unit/puppet/provider/mongodb_database/mongodb_spec.rb +++ b/spec/unit/puppet/provider/mongodb_database/mongodb_spec.rb @@ -38,7 +38,7 @@ tmp = Tempfile.new('test') mongodconffile = tmp.path allow(provider.class).to receive(:mongod_conf_file).and_return(mongodconffile) - allow(provider.class).to receive(:mongo_eval).with('db.getMongo().setReadPref("primaryPreferred");EJSON.stringify(db.getMongo().getDBs())').and_return(raw_dbs) + allow(provider.class).to receive(:mongo_eval).with('EJSON.stringify(db.getMongo().getDBs())').and_return(raw_dbs) allow(provider.class).to receive(:db_ismaster).and_return(true) end From 55c52f1176090cd1272643f8d8be45687c428e01 Mon Sep 17 00:00:00 2001 From: Hugo Haakseth Date: Tue, 16 Apr 2024 11:16:50 +0200 Subject: [PATCH 2/7] Fix sharding acceptance testing --- lib/puppet/provider/mongodb_shard/mongo.rb | 62 +------------------ spec/acceptance/sharding_spec.rb | 62 +++++++++++++------ .../provider/mongodb_shard/mongodb_spec.rb | 32 +++++----- 3 files changed, 61 insertions(+), 95 deletions(-) diff --git a/lib/puppet/provider/mongodb_shard/mongo.rb b/lib/puppet/provider/mongodb_shard/mongo.rb index 6347bf59b..b30602973 100644 --- a/lib/puppet/provider/mongodb_shard/mongo.rb +++ b/lib/puppet/provider/mongodb_shard/mongo.rb @@ -102,7 +102,7 @@ def self.shard_collection_details(obj, shard_name) def self.shard_properties(shard) properties = {} - output = mongo_command('sh.status()') + output = mongo_command('sh.status()')['value'] output['shards'].each do |s| next unless s['_id'] == shard @@ -118,7 +118,7 @@ def self.shard_properties(shard) end def self.shards_properties - output = mongo_command('sh.status()') + output = mongo_command('sh.status()')['value'] properties = if output['shards'].empty? [] else @@ -163,64 +163,6 @@ def self.mongo_command(command, host = nil, _retries = 4) retry end - # NOTE: (spredzy) : sh.status() - # does not return a json stream - # we jsonify it so it is easier - # to parse and deal with it - if command == 'sh.status()' - myarr = output.split("\n") - myarr.shift - myarr.pop - myarr.pop - final_stream = [] - prev_line = nil - in_shard_list = 0 - in_chunk = 0 - myarr.each do |line| - line.gsub!(%r{sharding version:}, '{ "sharding version":') - line.gsub!(%r{shards:}, ',"shards":[') - line.gsub!(%r{databases:}, '], "databases":[') - line.gsub!(%r{"clusterId" : ObjectId\("(.*)"\)}, '"clusterId" : "ObjectId(\'\1\')"') - line.gsub!(%r{\{ "_id" :}, ',{ "_id" :') if %r{_id} =~ prev_line - # Modification for shard - line = '' if line =~ %r{on :.*Timestamp} - if line =~ %r{_id} && in_shard_list == 1 - in_shard_list = 0 - last_line = final_stream.pop.strip - proper_line = "#{last_line}]}," - final_stream << proper_line - end - if line =~ %r{shard key} && in_shard_list == 1 - shard_name = final_stream.pop.strip - proper_line = ",{\"#{shard_name}\":" - final_stream << proper_line - end - if line =~ %r{shard key} && in_shard_list.zero? - in_shard_list = 1 - shard_name = final_stream.pop.strip - id_line = "#{final_stream.pop[0..-2]}, \"shards\": " - proper_line = "[{\"#{shard_name}\":" - final_stream << id_line - final_stream << proper_line - end - if in_chunk == 1 - in_chunk = 0 - line = "\"#{line.strip}\"}}" - end - in_chunk = 1 if line =~ %r{chunks} && in_chunk.zero? - line.gsub!(%r{shard key}, '{"shard key"') - line.gsub!(%r{chunks}, ',"chunks"') - final_stream << line unless line.empty? - prev_line = line - end - final_stream << ' ] }' if in_shard_list == 1 - final_stream << ' ] }' - output = final_stream.join("\n") - end - - # Hack to avoid non-json empty sets - output = '{}' if output == "null\n" - output.gsub!(%r{\s*}, '') JSON.parse(output) end end diff --git a/spec/acceptance/sharding_spec.rb b/spec/acceptance/sharding_spec.rb index 2c826234a..66c7c34ac 100644 --- a/spec/acceptance/sharding_spec.rb +++ b/spec/acceptance/sharding_spec.rb @@ -7,21 +7,41 @@ if hosts.length > 1 && supported_version?(default[:platform], repo_version) describe 'mongodb_shard resource' do + after :all do + pp = <<-EOS + class { 'mongodb::globals': + #{repo_ver_param} + } + -> class { 'mongodb::mongos' : + package_ensure => purged, + service_ensure => stopped + } + -> class { 'mongodb::server': + ensure => absent, + package_ensure => purged, + service_ensure => stopped + } + -> class { 'mongodb::client': + ensure => purged + } + EOS + + apply_manifest_on(hosts.reverse, pp, catch_failures: true) + end + it 'configures the shard server' do pp = <<-EOS class { 'mongodb::globals': #{repo_ver_param} } + -> class { 'mongodb::client': } -> class { 'mongodb::server': - bind_ip => '0.0.0.0', + bind_ip => ['0.0.0.0'], replset => 'foo', shardsvr => true, - }-> - mongodb_replset { 'foo' : - members => ["shard:27018"], } - if $::osfamily == 'RedHat' { - class { 'mongodb::client': } + -> mongodb_replset { 'foo' : + members => [#{hosts_as('shard').map { |x| "'#{x}:27018'" }.join(',')}], } EOS @@ -34,27 +54,29 @@ class { 'mongodb::client': } class { 'mongodb::globals': #{repo_ver_param} } + -> class { 'mongodb::client': } -> class { 'mongodb::server': - bind_ip => '0.0.0.0', + bind_ip => ['0.0.0.0'], + replset => 'conf', configsvr => true, - } -> - class { 'mongodb::mongos' : - configdb => ["router:27019"], - } -> - exec { '/bin/sleep 20' : - } -> - mongodb_shard { 'foo': - member => 'foo/shard:27018', - keys => [{'foo.toto' => {'name' => 1}}] } - if $::osfamily == 'RedHat' { - class { 'mongodb::client': } + -> mongodb_replset { 'conf' : + members => [#{hosts_as('router').map { |x| "'#{x}:27019'" }.join(',')}], + } + -> class { 'mongodb::mongos' : + configdb => ["conf/#{hosts_as('router').map { |x| "#{x}:27019" }.join(',')}"], + } + -> exec { '/bin/sleep 20' : + } + -> mongodb_shard { 'foo': + member => "foo/#{hosts_as('shard').map { |x| "#{x}:27018" }.join(',')}", + keys => [{'foo.toto' => {'name' => 1}}] } EOS - apply_manifest_on(hosts_as('router'), pp, catch_failures: true) + apply_manifest_on(hosts_as('router'), pp, catch_failures: true, trace: true) on(hosts_as('router'), 'mongosh --quiet --eval "EJSON.stringify(sh.status())"') do |r| - expect(r.stdout).to match %r{foo/shard:27018} + expect(r.stdout).to match %r{foo/#{hosts[0]}:27018} expect(r.stdout).to match %r{foo\.toto} end end diff --git a/spec/unit/puppet/provider/mongodb_shard/mongodb_spec.rb b/spec/unit/puppet/provider/mongodb_shard/mongodb_spec.rb index 865ed8d87..d749519cc 100644 --- a/spec/unit/puppet/provider/mongodb_shard/mongodb_spec.rb +++ b/spec/unit/puppet/provider/mongodb_shard/mongodb_spec.rb @@ -20,21 +20,23 @@ let(:raw_shards) do { - 'sharding version' => { - '_id' => 1, - 'version' => 4, - 'minCompatibleVersion' => 4, - 'currentVersion' => 5, - 'clusterId' => "ObjectId('548e9110f3aca177c94c5e49')" - }, - 'shards' => [ - { '_id' => 'rs_test', 'host' => 'rs_test/mongo1:27018' } - ], - 'databases' => [ - { '_id' => 'admin', 'partitioned' => false, 'primary' => 'config' }, - { '_id' => 'test', 'partitioned' => false, 'primary' => 'rs_test' }, - { '_id' => 'rs_test', 'partitioned' => true, 'primary' => 'rs_test' } - ] + 'value' => { + 'sharding version' => { + '_id' => 1, + 'version' => 4, + 'minCompatibleVersion' => 4, + 'currentVersion' => 5, + 'clusterId' => "ObjectId('548e9110f3aca177c94c5e49')" + }, + 'shards' => [ + { '_id' => 'rs_test', 'host' => 'rs_test/mongo1:27018' } + ], + 'databases' => [ + { '_id' => 'admin', 'partitioned' => false, 'primary' => 'config' }, + { '_id' => 'test', 'partitioned' => false, 'primary' => 'rs_test' }, + { '_id' => 'rs_test', 'partitioned' => true, 'primary' => 'rs_test' } + ] + } } end From f9957b3bf23079813ca32db0c32c5ff4fd0ba38c Mon Sep 17 00:00:00 2001 From: Hugo Haakseth Date: Wed, 17 Apr 2024 00:04:29 +0200 Subject: [PATCH 3/7] Use multinode setfile in ci workflow --- .github/workflows/ci.yml | 98 +++++++++++++++++++++++++++++++++++++--- .sync.yml | 1 + 2 files changed, 92 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1b263c33..f1cb8d2e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,7 @@ # Managed by modulesync - DO NOT EDIT # https://voxpupuli.org/docs/updating-files-managed-with-modulesync/ -name: CI +name: 'CI' on: pull_request: {} @@ -16,9 +16,93 @@ concurrency: cancel-in-progress: true jobs: - puppet: - name: Puppet - uses: voxpupuli/gha-puppet/.github/workflows/beaker.yml@v2 - with: - pidfile_workaround: 'false' - beaker_facter: 'mongodb_repo_version:MongoDB:4.4,5.0,6.0,7.0' + setup_matrix: + name: 'Static validations' + runs-on: ubuntu-latest + timeout-minutes: 40 + outputs: + puppet_unit_test_matrix: ${{ steps.get-outputs.outputs.puppet_unit_test_matrix }} + puppet_beaker_test_matrix: ${{ steps.get-outputs.outputs.puppet_beaker_test_matrix }} + env: + BUNDLE_WITHOUT: development:system_tests:release + PUPPET_GEM_VERSION: ">= 7.0" + steps: + - uses: actions/checkout@v4 + - name: Setup ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.2' + bundler-cache: true + - name: Run static validations + run: bundle exec rake validate lint check + - name: Run rake rubocop + run: bundle exec rake rubocop + - name: Setup Test Matrix + id: get-outputs + run: bundle exec metadata2gha --pidfile-workaround false + + unit: + needs: setup_matrix + runs-on: ubuntu-latest + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + include: ${{fromJson(needs.setup_matrix.outputs.puppet_unit_test_matrix)}} + env: + BUNDLE_WITHOUT: development:system_tests:release + PUPPET_GEM_VERSION: "~> ${{ matrix.puppet }}.0" + name: Puppet ${{ matrix.puppet }} (Ruby ${{ matrix.ruby }}) + steps: + - uses: actions/checkout@v4 + - name: Setup ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true + - name: Run tests + run: bundle exec rake parallel_spec + + acceptance: + needs: setup_matrix + runs-on: ubuntu-20.04 + env: + BEAKER_HYPERVISOR: docker + BUNDLE_WITHOUT: development:test:release + strategy: + fail-fast: false + matrix: + puppet: ${{fromJson(needs.setup_matrix.outputs.puppet_beaker_test_matrix)}} + mongodb: + - name: MongoDB 4.4 + value: '4.4' + - name: MongoDB 5.0 + value: '5.0' + - name: MongoDB 6.0 + value: '6.0' + - name: MongoDB 7.0 + value: '7.0' + name: ${{ matrix.puppet.name }} - ${{ matrix.mongodb.name }} + steps: + - uses: actions/checkout@v4 + - name: Setup ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.2' + bundler-cache: true + - name: Run tests + run: bundle exec rake beaker + env: + BEAKER_PUPPET_COLLECTION: ${{ matrix.puppet.env.BEAKER_PUPPET_COLLECTION }} + BEAKER_setfile: ${{ matrix.puppet.env.BEAKER_SETFILE }}shard.ma{hostname=${{ matrix.puppet.env.BEAKER_SETFILE }}-1-${{ matrix.puppet.env.BEAKER_PUPPET_COLLECTION }}.example.com}-${{ matrix.puppet.env.BEAKER_SETFILE }}slave,router.a{hostname=${{ matrix.puppet.env.BEAKER_SETFILE }}-2-${{ matrix.puppet.env.BEAKER_PUPPET_COLLECTION }}.example.com} + BEAKER_FACTER_mongodb_repo_version: ${{ matrix.mongodb.value }} + + tests: + needs: + - acceptance + runs-on: ubuntu-latest + name: Test suite + steps: + - run: echo Test suite completed + # explicitly run at the root dir. In case people set working-directory this will otherwise fail (because this jobs doesn't clone the repo and the subdir doesn't exist) + working-directory: '.' diff --git a/.sync.yml b/.sync.yml index ee3be25a5..5a3268aba 100644 --- a/.sync.yml +++ b/.sync.yml @@ -3,3 +3,4 @@ appveyor.yml: delete: true .github/workflows/ci.yml: beaker_facter: 'mongodb_repo_version:MongoDB:4.4,5.0,6.0,7.0' + unmanaged: true From 24bb055dc8a44e98590fb141e4232e0f81ab1306 Mon Sep 17 00:00:00 2001 From: Hugo Haakseth Date: Sun, 21 Apr 2024 00:58:14 +0200 Subject: [PATCH 4/7] Look at all active hosts to detect if replset init is successfully ended --- lib/puppet/provider/mongodb_replset/mongo.rb | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/puppet/provider/mongodb_replset/mongo.rb b/lib/puppet/provider/mongodb_replset/mongo.rb index e9d134c6c..d828a7ea6 100644 --- a/lib/puppet/provider/mongodb_replset/mongo.rb +++ b/lib/puppet/provider/mongodb_replset/mongo.rb @@ -284,13 +284,15 @@ def create_replica_set(alive_hosts) retry_sleep = 3 retry_limit.times do |n| - if db_ismaster(alive_hosts[0]['host'])['ismaster'] - Puppet.debug 'Replica set initialization has successfully ended' - return true - else - Puppet.debug "Waiting for replica initialization. Retry: #{n}" - sleep retry_sleep - next + alive_hosts.each do |alive_host| + if db_ismaster(alive_host['host'])['ismaster'] + Puppet.debug 'Replica set initialization has successfully ended' + return true + else + Puppet.debug "Waiting for replica initialization. Retry: #{n}" + sleep retry_sleep + next + end end end raise Puppet::Error, "rs.initiate() failed for replicaset #{name}" From 7f36dacfc80483da9911b1eb0834efe3039ddb1e Mon Sep 17 00:00:00 2001 From: Hugo Haakseth Date: Sun, 21 Apr 2024 22:44:00 +0200 Subject: [PATCH 5/7] replset integration tests: Manage server and replset in one puppet run. --- spec/acceptance/replset_spec.rb | 56 ++++++++++++++++----------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/spec/acceptance/replset_spec.rb b/spec/acceptance/replset_spec.rb index 4393ed56c..bb1568354 100644 --- a/spec/acceptance/replset_spec.rb +++ b/spec/acceptance/replset_spec.rb @@ -30,24 +30,22 @@ class { 'mongodb::globals': class { 'mongodb::globals': #{repo_ver_param} } + -> class { 'mongodb::client': } -> class { 'mongodb::server': bind_ip => ['0.0.0.0'], replset => 'test', } - -> class { 'mongodb::client': } + -> mongodb_replset { 'test': + members => [#{hosts.map { |x| "'#{x}:27017'" }.join(',')}], + } + EOS - apply_manifest_on(hosts.reverse, pp, catch_failures: true) - apply_manifest_on(hosts.reverse, pp, catch_changes: true) + apply_manifest_on(hosts, pp, catch_failures: true) + apply_manifest_on(hosts, pp, catch_changes: true) end it 'sets up the replset with puppet' do - pp = <<-EOS - mongodb_replset { 'test': - members => [#{hosts.map { |x| "'#{x}:27017'" }.join(',')}], - } - EOS - apply_manifest_on(hosts_as('master'), pp, catch_failures: true) on(hosts_as('master'), 'mongosh --quiet --eval "EJSON.stringify(rs.conf())"') do |r| expect(r.stdout).to match %r{#{hosts[0]}:27017} expect(r.stdout).to match %r{#{hosts[1]}:27017} @@ -101,12 +99,12 @@ class { 'mongodb::globals': class { 'mongodb::globals': #{repo_ver_param} } + -> class { 'mongodb::client': } -> class { 'mongodb::server': bind_ip => ['0.0.0.0'], replset => 'test', replset_members => [#{hosts.map { |x| "'#{x}:27017'" }.join(',')}], } - -> class { 'mongodb::client': } EOS apply_manifest_on(hosts, pp, catch_failures: true) @@ -176,6 +174,7 @@ class { 'mongodb::globals': class { 'mongodb::globals': #{repo_ver_param} } + -> class { 'mongodb::client': } -> class { 'mongodb::server': admin_username => 'admin', admin_password => 'password', @@ -201,30 +200,29 @@ class { 'mongodb::globals': nc1ohyB0lNt8lHf1U00mtgDSV3fwo5LkwhRi6d+bDBTL/C6MZETMLdyCqDlTdUWG YXIsJ0gYcu9XG3mx10LbdPJvxSMg', } - -> class { 'mongodb::client': } + -> mongodb_replset { 'test': + members => [#{hosts.map { |x| "'#{x}:27017'" }.join(',')}], + } + + unless $facts['mongodb_is_master'] == 'false' { # lint:ignore:quoted_booleans + mongodb_user { "User admin on db admin": + ensure => present, + password_hash => mongodb_password('admin', 'password'), + username => 'admin', + database => 'admin', + roles => ['userAdmin', 'readWrite', 'dbAdmin', 'dbAdminAnyDatabase', 'readAnyDatabase', + 'readWriteAnyDatabase', 'userAdminAnyDatabase', 'clusterAdmin', + 'clusterManager', 'clusterMonitor', 'hostManager', 'root', 'restore',], + require => Mongodb_replset['test'], + } + } EOS - apply_manifest_on(hosts.reverse, pp, catch_failures: true) - apply_manifest_on(hosts.reverse, pp, catch_changes: true) + apply_manifest_on(hosts, pp, catch_failures: true) + apply_manifest_on(hosts, pp, catch_changes: true) end it 'sets up the replset with puppet' do - pp = <<~EOS - mongodb_replset { 'test': - members => [#{hosts.map { |x| "'#{x}:27017'" }.join(',')}], - } - -> mongodb_user { "User admin on db admin": - ensure => present, - password_hash => mongodb_password('admin', 'password'), - username => 'admin', - database => 'admin', - roles => ['userAdmin', 'readWrite', 'dbAdmin', 'dbAdminAnyDatabase', 'readAnyDatabase', - 'readWriteAnyDatabase', 'userAdminAnyDatabase', 'clusterAdmin', - 'clusterManager', 'clusterMonitor', 'hostManager', 'root', 'restore',], - } - EOS - apply_manifest_on(hosts_as('master'), pp, catch_failures: true) - apply_manifest_on(hosts_as('master'), pp, catch_changes: true) on(hosts_as('master'), 'mongosh --quiet --eval "load(\'/root/.mongoshrc.js\');EJSON.stringify(rs.conf())"') do |r| expect(r.stdout).to match %r{#{hosts[0]}:27017} expect(r.stdout).to match %r{#{hosts[1]}:27017} From f3e59ef1239564c59909303b152ce725b599493d Mon Sep 17 00:00:00 2001 From: Hugo Haakseth Date: Mon, 22 Apr 2024 20:03:30 +0200 Subject: [PATCH 6/7] test modified metadata2gha --- .github/workflows/ci.yml | 130 +++++++-------------------------------- .sync.yml | 2 +- README.md | 1 + 3 files changed, 25 insertions(+), 108 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1cb8d2e8..63bd89837 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,108 +1,24 @@ --- -# Managed by modulesync - DO NOT EDIT -# https://voxpupuli.org/docs/updating-files-managed-with-modulesync/ - -name: 'CI' - -on: - pull_request: {} - push: - branches: - - main - - master - -concurrency: - group: ${{ github.ref_name }} - cancel-in-progress: true - -jobs: - setup_matrix: - name: 'Static validations' - runs-on: ubuntu-latest - timeout-minutes: 40 - outputs: - puppet_unit_test_matrix: ${{ steps.get-outputs.outputs.puppet_unit_test_matrix }} - puppet_beaker_test_matrix: ${{ steps.get-outputs.outputs.puppet_beaker_test_matrix }} - env: - BUNDLE_WITHOUT: development:system_tests:release - PUPPET_GEM_VERSION: ">= 7.0" - steps: - - uses: actions/checkout@v4 - - name: Setup ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: '3.2' - bundler-cache: true - - name: Run static validations - run: bundle exec rake validate lint check - - name: Run rake rubocop - run: bundle exec rake rubocop - - name: Setup Test Matrix - id: get-outputs - run: bundle exec metadata2gha --pidfile-workaround false - - unit: - needs: setup_matrix - runs-on: ubuntu-latest - timeout-minutes: 40 - strategy: - fail-fast: false - matrix: - include: ${{fromJson(needs.setup_matrix.outputs.puppet_unit_test_matrix)}} - env: - BUNDLE_WITHOUT: development:system_tests:release - PUPPET_GEM_VERSION: "~> ${{ matrix.puppet }}.0" - name: Puppet ${{ matrix.puppet }} (Ruby ${{ matrix.ruby }}) - steps: - - uses: actions/checkout@v4 - - name: Setup ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ matrix.ruby }} - bundler-cache: true - - name: Run tests - run: bundle exec rake parallel_spec - - acceptance: - needs: setup_matrix - runs-on: ubuntu-20.04 - env: - BEAKER_HYPERVISOR: docker - BUNDLE_WITHOUT: development:test:release - strategy: - fail-fast: false - matrix: - puppet: ${{fromJson(needs.setup_matrix.outputs.puppet_beaker_test_matrix)}} - mongodb: - - name: MongoDB 4.4 - value: '4.4' - - name: MongoDB 5.0 - value: '5.0' - - name: MongoDB 6.0 - value: '6.0' - - name: MongoDB 7.0 - value: '7.0' - name: ${{ matrix.puppet.name }} - ${{ matrix.mongodb.name }} - steps: - - uses: actions/checkout@v4 - - name: Setup ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: '3.2' - bundler-cache: true - - name: Run tests - run: bundle exec rake beaker - env: - BEAKER_PUPPET_COLLECTION: ${{ matrix.puppet.env.BEAKER_PUPPET_COLLECTION }} - BEAKER_setfile: ${{ matrix.puppet.env.BEAKER_SETFILE }}shard.ma{hostname=${{ matrix.puppet.env.BEAKER_SETFILE }}-1-${{ matrix.puppet.env.BEAKER_PUPPET_COLLECTION }}.example.com}-${{ matrix.puppet.env.BEAKER_SETFILE }}slave,router.a{hostname=${{ matrix.puppet.env.BEAKER_SETFILE }}-2-${{ matrix.puppet.env.BEAKER_PUPPET_COLLECTION }}.example.com} - BEAKER_FACTER_mongodb_repo_version: ${{ matrix.mongodb.value }} - - tests: - needs: - - acceptance - runs-on: ubuntu-latest - name: Test suite - steps: - - run: echo Test suite completed - # explicitly run at the root dir. In case people set working-directory this will otherwise fail (because this jobs doesn't clone the repo and the subdir doesn't exist) - working-directory: '.' + # Managed by modulesync - DO NOT EDIT + # https://voxpupuli.org/docs/updating-files-managed-with-modulesync/ + + name: CI + + on: + pull_request: {} + push: + branches: + - main + - master + + concurrency: + group: ${{ github.ref_name }} + cancel-in-progress: true + + jobs: + puppet: + name: Puppet + uses: StatensPensjonskasse/gha-puppet/.github/workflows/beaker.yml@add-metadata2gha-option-beaker-hosts + with: + pidfile_workaround: 'false' + beaker_facter: 'mongodb_repo_version:MongoDB:4.4,5.0,6.0,7.0' diff --git a/.sync.yml b/.sync.yml index 5a3268aba..af42cf39e 100644 --- a/.sync.yml +++ b/.sync.yml @@ -2,5 +2,5 @@ appveyor.yml: delete: true .github/workflows/ci.yml: + beaker_hosts: 'host1:shard.ma;host2:slave,router.a' beaker_facter: 'mongodb_repo_version:MongoDB:4.4,5.0,6.0,7.0' - unmanaged: true diff --git a/README.md b/README.md index c6af068d8..d151d1210 100644 --- a/README.md +++ b/README.md @@ -158,3 +158,4 @@ We would like to thank everyone who has contributed issues and pull requests to A complete list of contributors can be found on the [GitHub Contributor Graph](https://github.com/voxpupuli/puppet-mongodb/graphs/contributors) for the [puppet-mongodb module](https://github.com/voxpupuli/puppet-mongodb). + From e5882ee09fdb21c9f0e896cc683e479302b31e19 Mon Sep 17 00:00:00 2001 From: Hugo Haakseth Date: Thu, 25 Apr 2024 17:06:56 +0200 Subject: [PATCH 7/7] Add beaker_hosts to ci.yml --- .github/workflows/ci.yml | 1 + README.md | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63bd89837..e9bc18b88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,4 +21,5 @@ uses: StatensPensjonskasse/gha-puppet/.github/workflows/beaker.yml@add-metadata2gha-option-beaker-hosts with: pidfile_workaround: 'false' + beaker_hosts: 'host1:shard.ma;host2:slave,router.a' beaker_facter: 'mongodb_repo_version:MongoDB:4.4,5.0,6.0,7.0' diff --git a/README.md b/README.md index d151d1210..c6af068d8 100644 --- a/README.md +++ b/README.md @@ -158,4 +158,3 @@ We would like to thank everyone who has contributed issues and pull requests to A complete list of contributors can be found on the [GitHub Contributor Graph](https://github.com/voxpupuli/puppet-mongodb/graphs/contributors) for the [puppet-mongodb module](https://github.com/voxpupuli/puppet-mongodb). -