From 6cb7e8fcaa341ff4f46e91f383d4906e833e300c Mon Sep 17 00:00:00 2001 From: jonathannewman Date: Tue, 20 Jun 2023 14:53:00 -0700 Subject: [PATCH 01/27] (PDB-5636) update ezbake to 2.4.2 This updates lein-ezbake to 2.4.2 to add build target support for ubuntu 2204 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 0a775a97c2..630a90e810 100644 --- a/project.clj +++ b/project.clj @@ -312,7 +312,7 @@ ;; in the final package. [puppetlabs/puppetdb ~pdb-version :exclusions [com.zaxxer/HikariCP]]] :name "puppetdb" - :plugins [[puppetlabs/lein-ezbake "2.4.1"]]} + :plugins [[puppetlabs/lein-ezbake "2.4.2"]]} :testutils {:source-paths ^:replace ["test"] :resource-paths ^:replace [] ;; Something else may need adjustment, but From 327cbf4e1e4d8d047f9a791fe98b5e599c637bcb Mon Sep 17 00:00:00 2001 From: jonathannewman Date: Mon, 26 Jun 2023 13:43:32 -0700 Subject: [PATCH 02/27] (PDB-5636) add ubuntu 2204 as supported platform This adds ubuntu 2204/jammy to the set of supported puppetdb versions starting now. --- acceptance/helper.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/acceptance/helper.rb b/acceptance/helper.rb index 131121ba6c..286b4cd7d9 100644 --- a/acceptance/helper.rb +++ b/acceptance/helper.rb @@ -284,6 +284,10 @@ def is_focal() return test_config[:os_families].has_key? 'ubuntu2004-64-1' end + def is_jammy() + return test_config[:os_families].has_key? 'ubuntu2204-64-1' + end + def is_buster() return test_config[:os_families].has_key? 'debian10-64-1' end @@ -339,7 +343,7 @@ def puppet_repo_version(platform_version, install_type, nightly) # supported version of PuppetDB. Its version must be available in the # platform version returned by puppet_repo_version above def oldest_supported - # account for bionic/rhel8 not having build before certian versions + # account for bionic/rhel8 not having build before certain versions if is_bullseye '7.9.0' elsif is_el8 @@ -350,6 +354,8 @@ def oldest_supported '6.7.0' elsif is_focal '6.12.0' + elsif is_jammy + '7.13.2' else '6.0.0' end @@ -415,6 +421,8 @@ def get_package_version(host, version = nil) "#{version}bionic" elsif host['platform'].include?('ubuntu-20.04') "#{version}focal" + elsif host['platform'].include?('ubuntu-22.04') + "#{version}jammy" elsif host['platform'].include?('debian-10') "#{version}buster" elsif host['platform'].include?('debian-11') From 6f278fac486c3dc97e3eaad83344460002e9fddc Mon Sep 17 00:00:00 2001 From: Josh Partlow Date: Wed, 28 Jun 2023 23:32:09 +0000 Subject: [PATCH 03/27] (maint) Don't print null user in disconnecting message I think this carried over from the last refactor; since user is null here, we end up printing "Disconnecting all null connections" in the log. Changed it to non-migrator, since I believe that is effectively what we are doing here. --- src/puppetlabs/puppetdb/scf/migrate.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/puppetlabs/puppetdb/scf/migrate.clj b/src/puppetlabs/puppetdb/scf/migrate.clj index 411e55efba..74b7281b00 100644 --- a/src/puppetlabs/puppetdb/scf/migrate.clj +++ b/src/puppetlabs/puppetdb/scf/migrate.clj @@ -2555,8 +2555,8 @@ (if-not user (do (log/info - (trs "Disconnecting all {0} connections to {1} database before migrating" - user db-name)) + (trs "Disconnecting all non-migrator connections to {0} database before migrating" + db-name)) (doseq [user users] ;; Because the revoke may not actually produce an error when ;; it doesn't work. From 7ef985259c1bfe91e7315985d443ad4ce3b54f1f Mon Sep 17 00:00:00 2001 From: Josh Partlow Date: Wed, 28 Jun 2023 23:33:59 +0000 Subject: [PATCH 04/27] (PE-36120) Do not kill connections when no migrations are pending Previous behavior would cause compilers to be disconnected from postgresql ever time the primary's pe-puppetdb was restarted. This was because the restart invokes the migration code path, which, in PE with a migrator-account set leads to connection blocking even when the migration itself is ultimately a noop. Now it tests whether any migrations are pending before calling call-with-connections-blocked-during-migration. This change also means that update-schema will return nil in this case, and initialize-schema will return false, which is a change from previous noop migrations where initialize-schema would have returned true. --- src/puppetlabs/puppetdb/scf/migrate.clj | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/puppetlabs/puppetdb/scf/migrate.clj b/src/puppetlabs/puppetdb/scf/migrate.clj index 74b7281b00..7f0a9dafcb 100644 --- a/src/puppetlabs/puppetdb/scf/migrate.clj +++ b/src/puppetlabs/puppetdb/scf/migrate.clj @@ -2617,16 +2617,17 @@ tables))] (if-not write-user (migrate) - (call-with-connections-blocked-during-migration - db-name - (distinct [read-user write-user]) - (fn [] - ;; So new tables, etc. are owned by the write-user - (jdbc/do-commands (str "set role " (jdbc/double-quote write-user))) - (try! - (migrate) - (finally - (jdbc/do-commands (str "set role " (jdbc/double-quote orig-user)))))))))) + (when-not (empty? (pending-migrations)) + (call-with-connections-blocked-during-migration + db-name + (distinct [read-user write-user]) + (fn [] + ;; So new tables, etc. are owned by the write-user + (jdbc/do-commands (str "set role " (jdbc/double-quote write-user))) + (try! + (migrate) + (finally + (jdbc/do-commands (str "set role " (jdbc/double-quote orig-user))))))))))) (defn initialize-schema "Ensures the database is migrated to the latest version, and returns From b2dfba2ef5d3b5ed3310dae9cf3e81ceb15f91bd Mon Sep 17 00:00:00 2001 From: Jenkins CI Date: Thu, 29 Jun 2023 20:54:08 +0000 Subject: [PATCH 05/27] (i18n) Update strings in puppetdb.pot file --- locales/puppetdb.pot | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/locales/puppetdb.pot b/locales/puppetdb.pot index 29b13ad4ad..09196ef6a6 100644 --- a/locales/puppetdb.pot +++ b/locales/puppetdb.pot @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: puppetlabs.puppetdb \n" -"X-Git-Ref: 1a530198597b03481c6b50fa1e4977d7dcd82bea\n" +"X-Git-Ref: f768ede7ed7251d1a06dd027462d2ec9e3392ccd\n" "Report-Msgid-Bugs-To: docs@puppet.com\n" "POT-Creation-Date: \n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" @@ -1593,7 +1593,8 @@ msgid "Creating additional index `catalog_resources_file_trgm`" msgstr "" #: src/puppetlabs/puppetdb/scf/migrate.clj -msgid "Disconnecting all {0} connections to {1} database before migrating" +msgid "" +"Disconnecting all non-migrator connections to {0} database before migrating" msgstr "" #: src/puppetlabs/puppetdb/scf/migrate.clj From 8ce3cae591019b4e8262a7af9ab32a9845df6191 Mon Sep 17 00:00:00 2001 From: Jenkins CI Date: Thu, 6 Jul 2023 18:17:57 +0000 Subject: [PATCH 06/27] Set clj-parent=5.3.8 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 630a90e810..b8813f7963 100644 --- a/project.clj +++ b/project.clj @@ -1,6 +1,6 @@ (def pdb-version "7.13.2-SNAPSHOT") -(def clj-parent-version "5.3.7") +(def clj-parent-version "5.3.8") (defn true-in-env? [x] (#{"true" "yes" "1"} (System/getenv x))) From b8c8364e1380a67d3156e37e1bcc18fc935efba6 Mon Sep 17 00:00:00 2001 From: Jenkins CI Date: Tue, 18 Jul 2023 16:03:32 +0000 Subject: [PATCH 07/27] Set clj-parent=5.3.9 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index b8813f7963..84fa4adf7b 100644 --- a/project.clj +++ b/project.clj @@ -1,6 +1,6 @@ (def pdb-version "7.13.2-SNAPSHOT") -(def clj-parent-version "5.3.8") +(def clj-parent-version "5.3.9") (defn true-in-env? [x] (#{"true" "yes" "1"} (System/getenv x))) From f37a5dd6a9764260e021b5a7571be33b59743b7c Mon Sep 17 00:00:00 2001 From: Austin Blatt Date: Thu, 4 May 2023 14:36:13 -0700 Subject: [PATCH 08/27] (PDB-5643) Use recursive PQL rules for and/or Change PQL expr-or and expr-and grammar rules to use a recursive definition. By using instaparse's `{ }` operator (meaning zero or more) in self-referential rules the grammar is ambiguous. a = 1 or b = 2 or c = 3 can be parsed as something that looks like [:or a = 1 b = 2 c = 3] Or it could be parsed as [:or a = 1 [:or b = 2 c = 3]] Both are valid given the self-referential grammar rule, but as clauses compound the number of valid parsings grows exponentially significantly affecting performance. As few as 16 or clauses can cause an OOM with 1GB heap. The recursive grammar definition means only one (the second example above) version is valid. Every and/or operator will have two children. --- resources/puppetlabs/puppetdb/pql/pql-grammar.ebnf | 4 ++-- test/puppetlabs/puppetdb/pql/parser_test.clj | 4 ++-- test/puppetlabs/puppetdb/pql_test.clj | 9 +++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/resources/puppetlabs/puppetdb/pql/pql-grammar.ebnf b/resources/puppetlabs/puppetdb/pql/pql-grammar.ebnf index da350c57e7..d65aff7fc0 100644 --- a/resources/puppetlabs/puppetdb/pql/pql-grammar.ebnf +++ b/resources/puppetlabs/puppetdb/pql/pql-grammar.ebnf @@ -52,8 +52,8 @@ extract = , [], [extractfields], [], = expr-or; -expr-or = expr-and { , or, , expr-or }; -expr-and = expr-not { , and, , expr-and }; +expr-or = expr-and | expr-and, , or, , expr-or; +expr-and = expr-not | expr-not, , and, , expr-and; expr-not = ( not, [], expr-not ) | expr-rest; = ( , [], expression, [], ) | condexpression | condexpnull | subquery; diff --git a/test/puppetlabs/puppetdb/pql/parser_test.clj b/test/puppetlabs/puppetdb/pql/parser_test.clj index 134f3a8e95..85db1fb62c 100644 --- a/test/puppetlabs/puppetdb/pql/parser_test.clj +++ b/test/puppetlabs/puppetdb/pql/parser_test.clj @@ -285,11 +285,11 @@ [:condexpression [:field "d"] "=" [:integer "4"]]] [:expr-and [:expr-not - [:condexpression [:field "a"] "=" [:integer "1"]]]]]] + [:condexpression [:field "a"] "=" [:integer "1"]]]]] [:expr-or [:expr-and [:expr-not - [:condexpression [:field "b"] "=" [:integer "2"]]]]]]] + [:condexpression [:field "b"] "=" [:integer "2"]]]]]]]] "(c = 3 or d = 4) and (a = 1 or b = 2)" [[:expr-or diff --git a/test/puppetlabs/puppetdb/pql_test.clj b/test/puppetlabs/puppetdb/pql_test.clj index 99fc0554b9..1358c607a7 100644 --- a/test/puppetlabs/puppetdb/pql_test.clj +++ b/test/puppetlabs/puppetdb/pql_test.clj @@ -120,10 +120,11 @@ ["from" "nodes" ["or" ["=" "a" 1] - ["and" - ["=" "b" 2] - ["=" "c" 3]] - ["=" "d" 4]]] + ["or" + ["and" + ["=" "b" 2] + ["=" "c" 3]] + ["=" "d" 4]]]] "nodes { a = 1 or b = 2 and (c = 3 or d = 4) }" ["from" "nodes" From 8640b863ed0b7b88c82824cca3346554a41a7a3d Mon Sep 17 00:00:00 2001 From: Josh Partlow Date: Tue, 18 Jul 2023 19:51:02 +0000 Subject: [PATCH 09/27] (PDB-5643) Add a regression test for the parse --- test/puppetlabs/puppetdb/pql/parser_test.clj | 32 ++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/puppetlabs/puppetdb/pql/parser_test.clj b/test/puppetlabs/puppetdb/pql/parser_test.clj index 85db1fb62c..b1b8e8139f 100644 --- a/test/puppetlabs/puppetdb/pql/parser_test.clj +++ b/test/puppetlabs/puppetdb/pql/parser_test.clj @@ -324,6 +324,38 @@ "a=1 b=2" "")) +;; PDB-5643 Check that a PQL expression with many OR clauses parses in reasonable time. +(deftest test-expression-depth-performance + (let [expression "(certname = 'foo22' or + certname = 'foo21' or + certname = 'foo20' or + certname = 'foo19' or + certname = 'foo18' or + certname = 'foo17' or + certname = 'foo16' or + certname = 'foo15' or + certname = 'foo14' or + certname = 'foo13' or + certname = 'foo12' or + certname = 'foo11' or + certname = 'foo10' or + certname = 'foo9' or + certname = 'foo8' or + certname = 'foo7' or + certname = 'foo6' or + certname = 'foo5' or + certname = 'foo4' or + certname = 'foo3' or + certname = 'foo2' or + certname = 'foo1')" + start-time (System/currentTimeMillis) + expected-duration 500] + (parse expression :start :expression) + (let [duration-millis (- (System/currentTimeMillis) start-time)] + (is + (< duration-millis expected-duration) + (format "(PDB-5643) Parsing a PQL expression with many OR clauses took %s milliseconds, which is longer than the alloted %s" duration-millis expected-duration))))) + (deftest test-subquery (are [in expected] (= (parse in :start :subquery) expected) "nodes{}" From 25e7e7eff485b937eef0b7543c73cf66640a375a Mon Sep 17 00:00:00 2001 From: Josh Partlow Date: Tue, 18 Jul 2023 20:03:21 +0000 Subject: [PATCH 10/27] (PDB-5643) Transform back to flattened OR AST expressions The patch to the pql-grammar.ebnf took care of the ambiguity causing exponential parsing of deep OR clauses, but produces a nested AST form that differs from the flat [or 1 2 3 4] structure that was previously being handed off to the query engine. Nick Lewis had suggested an additional patch to transform.clj which would restructure the AST back to the form that the PQL parser was previously producing so that the rest of the system isn't blind-sided by a differing query structure. Since the issue we were fixing was in the initial parse of the PQL, taken together, these patches should fix the slow parse time without introducing any other novel behavior in the query engine. --- src/puppetlabs/puppetdb/pql/transform.clj | 16 ++++++++++++++-- test/puppetlabs/puppetdb/pql_test.clj | 13 ++++++------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/puppetlabs/puppetdb/pql/transform.clj b/src/puppetlabs/puppetdb/pql/transform.clj index ef321471e7..797f146577 100644 --- a/src/puppetlabs/puppetdb/pql/transform.clj +++ b/src/puppetlabs/puppetdb/pql/transform.clj @@ -33,13 +33,25 @@ ;; Single arg? collapse ([data] data) ;; Multiple args? turn it into an or statement - ([data & args] (vec (concat ["or" data] args)))) + ([data & args] + (->> (cons data args) + (mapcat #(if (and (vector? %) (= (first %) "or")) + (rest %) + [%])) + (cons "or") + (vec)))) (defn transform-expr-and ;; Single arg? collapse ([data] data) ;; Multiple args? turn it into an and statement - ([data & args] (vec (concat ["and" data] args)))) + ([data & args] + (->> (cons data args) + (mapcat #(if (and (vector? %) (= (first %) "and")) + (rest %) + [%])) + (cons "and") + (vec)))) (defn transform-expr-not ;; Single arg? Just collapse the :expr-not and pass back the data, diff --git a/test/puppetlabs/puppetdb/pql_test.clj b/test/puppetlabs/puppetdb/pql_test.clj index 1358c607a7..11452c5af5 100644 --- a/test/puppetlabs/puppetdb/pql_test.clj +++ b/test/puppetlabs/puppetdb/pql_test.clj @@ -3,8 +3,8 @@ [puppetlabs.puppetdb.pql :as pql])) (deftest test-pql->ast - (are [pql ast] (= (first (pql/pql->ast pql)) - ast) + (are [pql ast] (= ast + (first (pql/pql->ast pql))) ;; Some basic comparisons @@ -120,11 +120,10 @@ ["from" "nodes" ["or" ["=" "a" 1] - ["or" - ["and" - ["=" "b" 2] - ["=" "c" 3]] - ["=" "d" 4]]]] + ["and" + ["=" "b" 2] + ["=" "c" 3]] + ["=" "d" 4]]] "nodes { a = 1 or b = 2 and (c = 3 or d = 4) }" ["from" "nodes" From c016d37d287feb420b2c47715edfae2576f48657 Mon Sep 17 00:00:00 2001 From: Austin Blatt Date: Wed, 28 Jun 2023 15:06:37 -0700 Subject: [PATCH 11/27] (maint) Add clj-yaml as a dev dependency This was removed from trapperkeeper due to a CVE, but it is used in our testing. (Originally this was in main, but the same adjustment to tk removing clj-yaml as a dependency made it's way into clj-parent 5.x and we need this in puppetdb 7.x now as well.) --- project.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/project.clj b/project.clj index 84fa4adf7b..e458104907 100644 --- a/project.clj +++ b/project.clj @@ -79,6 +79,7 @@ :exclusions [com.andrewmcveigh/cljs-time]] [riddley "0.1.12"] [io.forward/yaml "1.0.5"] + [clj-commons/clj-yaml "1.0.26"] ;; Only needed for :integration tests [puppetlabs/trapperkeeper-filesystem-watcher nil]] From 3428ea7fb727d82c8b61eac660baafd33c4e06df Mon Sep 17 00:00:00 2001 From: rileynewton <95654936+rileynewton@users.noreply.github.com> Date: Thu, 20 Jul 2023 14:17:22 -0700 Subject: [PATCH 12/27] (PE-36184) Update ezbake to 2.5.0 in prep for logback 1.3.x bump --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index e458104907..d6100f77fa 100644 --- a/project.clj +++ b/project.clj @@ -313,7 +313,7 @@ ;; in the final package. [puppetlabs/puppetdb ~pdb-version :exclusions [com.zaxxer/HikariCP]]] :name "puppetdb" - :plugins [[puppetlabs/lein-ezbake "2.4.2"]]} + :plugins [[puppetlabs/lein-ezbake "2.5.0"]]} :testutils {:source-paths ^:replace ["test"] :resource-paths ^:replace [] ;; Something else may need adjustment, but From d9dbd55a49a4ba8fab0864b119428d640a5c3aff Mon Sep 17 00:00:00 2001 From: jonathannewman Date: Mon, 24 Jul 2023 14:15:30 -0700 Subject: [PATCH 13/27] (maint) update ezbake to 2.5.2 This updates ezbake to 2.5.2, which does two things: * it enables builds of the el9 platform * it adds in tzdata-java as a dependency on the el platforms tzdata-java was strangely removed as a dependency from openjdk, so this adds it in as a dependendency to ensure we don't release a broken package. --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index d6100f77fa..881778066c 100644 --- a/project.clj +++ b/project.clj @@ -313,7 +313,7 @@ ;; in the final package. [puppetlabs/puppetdb ~pdb-version :exclusions [com.zaxxer/HikariCP]]] :name "puppetdb" - :plugins [[puppetlabs/lein-ezbake "2.5.0"]]} + :plugins [[puppetlabs/lein-ezbake "2.5.2"]]} :testutils {:source-paths ^:replace ["test"] :resource-paths ^:replace [] ;; Something else may need adjustment, but From 1081492f4ef5190bc6ed2bd34d5ba72247a33301 Mon Sep 17 00:00:00 2001 From: Jenkins CI Date: Mon, 24 Jul 2023 21:19:13 +0000 Subject: [PATCH 14/27] Set clj-parent=5.4.0 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index d6100f77fa..7725ce9d56 100644 --- a/project.clj +++ b/project.clj @@ -1,6 +1,6 @@ (def pdb-version "7.13.2-SNAPSHOT") -(def clj-parent-version "5.3.9") +(def clj-parent-version "5.4.0") (defn true-in-env? [x] (#{"true" "yes" "1"} (System/getenv x))) From 9cc61bacecda21c50a538a386d357e72018853dc Mon Sep 17 00:00:00 2001 From: Rob Browning Date: Mon, 9 Jan 2023 16:32:46 -0600 Subject: [PATCH 15/27] (PDB-5572) config-puppet-test-ref: avoid bundle --path and --without They've been deprecated, "set --local path/without" instead. --- ext/bin/config-puppet-test-ref | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ext/bin/config-puppet-test-ref b/ext/bin/config-puppet-test-ref index cf314abda6..51d56ee894 100755 --- a/ext/bin/config-puppet-test-ref +++ b/ext/bin/config-puppet-test-ref @@ -49,7 +49,9 @@ case "$#" in esac # Install and update Ruby dependencies from top-level Gemfile -bundle install --without acceptance --path vendor/bundle --retry=10 +bundle config set --local path vendor/bundle +bundle config set --local without acceptance +bundle install --retry=10 bundle update # Print out info on Puppet dependency bundle info puppet || true # Not all versions appear to have info @@ -59,6 +61,6 @@ bundle info puppet || true # Not all versions appear to have info cd vendor # This will something like .../vendor/bundle/ruby/2.7.0/bundler/gems/puppet-ae5379e03311 # It is a Puppet git repository installed by bundler -puppet_path="$(bundle show puppet)" +puppet_path="$(bundle info puppet --path)" # Create a symlink at `vendor/puppet` that points to Puppet repository "$top/ext/bin/symlink-relative-to" "$puppet_path" puppet From d7e619df148f52f3d9661cfd8c2c2f7e796d6ca9 Mon Sep 17 00:00:00 2001 From: Rob Browning Date: Mon, 9 Jan 2023 16:30:33 -0600 Subject: [PATCH 16/27] (PDB-5572) Run puppetserver in its own JVM in :integration tests This avoids conflating depdendencies, so that we should use exactly the set for each server set that'll be in running when deployed. --- ext/bin/config-puppetserver-test-ref | 1 + .../puppetdb/integration/fixtures.clj | 56 +++++++++++++++---- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/ext/bin/config-puppetserver-test-ref b/ext/bin/config-puppetserver-test-ref index e86064881b..b0ef2e0841 100755 --- a/ext/bin/config-puppetserver-test-ref +++ b/ext/bin/config-puppetserver-test-ref @@ -72,6 +72,7 @@ fi # If we're not testing against the git tree (say in jenkins), don't install. if test -z "$PDB_NO_PUPPETSERVER_INSTALL"; then lein install + lein uberjar fi # Cache Puppet Server version in a file in ext/test-conf directory dep_ver="$(lein-pprint :version)" diff --git a/test/puppetlabs/puppetdb/integration/fixtures.clj b/test/puppetlabs/puppetdb/integration/fixtures.clj index 3eb449f848..c4affe51b4 100644 --- a/test/puppetlabs/puppetdb/integration/fixtures.clj +++ b/test/puppetlabs/puppetdb/integration/fixtures.clj @@ -13,13 +13,14 @@ [puppetlabs.puppetdb.testutils.db :as dbutils] [puppetlabs.puppetdb.testutils.services :as svc-utils] [puppetlabs.trapperkeeper.app :as tk-app] - [puppetlabs.trapperkeeper.bootstrap :as tk-bootstrap] [puppetlabs.trapperkeeper.config :as tk-config] [puppetlabs.trapperkeeper.testutils.bootstrap :as tkbs] [yaml.core :as yaml] [puppetlabs.puppetdb.time :as time] [puppetlabs.puppetdb.utils :as utils]) - (:import [com.typesafe.config ConfigValueFactory])) + (:import + (com.typesafe.config ConfigValueFactory) + (java.lang ProcessBuilder ProcessBuilder$Redirect))) (defprotocol TestServer (server-info [this])) @@ -154,13 +155,13 @@ ;;; Puppet Server fixture -(defrecord PuppetServerTestServer [info-map files-to-cleanup app] +(defrecord PuppetServerTestServer [info-map files-to-cleanup process] TestServer (server-info [_] info-map) java.lang.AutoCloseable (close [_] - (tk-app/stop app) + (doto process .destroy .waitFor) ;; Assumes destroy sends a SIGTERM (doseq [f files-to-cleanup] (fs/delete f)))) (def dev-config-file "./test-resources/puppetserver/puppetserver.conf") @@ -201,6 +202,21 @@ (clojure.string/join ","))}} (or overrides {}))))) +(defn- wait-for-server + [port timeout-ms] + (loop [n (/ timeout-ms 100)] + (when (neg? n) + (throw (ex-info (str "server not ready within " timeout-ms "ms") {}))) + (let [res (try + (svc-utils/get-ssl (str "https://localhost:" port "/status/v1/services")) + (catch java.net.ConnectException _ + ::nope))] + (cond + (= res ::nope) (do (Thread/sleep 100) (recur (dec n))) + (= 200 (:status res)) true + :else (throw (ex-info "Unexpected result from status endpoint while waiting" + res)))))) + (defn run-puppet-server-as [node-name pdb-servers config-overrides] (let [puppetdb-conf (io/file "target/puppetserver/master-conf/puppetdb.conf") puppet-conf (io/file "target/puppetserver/master-conf/puppet.conf") @@ -224,21 +240,37 @@ (write-puppetdb-terminus-config pdb-servers puppetdb-conf terminus-config-overrides) - (let [services (tk-bootstrap/parse-bootstrap-config! dev-bootstrap-file) - tmp-conf (ks/temp-file "puppetserver" ".conf") - _ (fs/copy dev-config-file tmp-conf) - port (svc-utils/open-port-num) - config (-> (tk-config/load-config (.getPath tmp-conf)) + (let [port (svc-utils/open-port-num) + config (-> (tk-config/load-config dev-config-file) (merge puppetserver-config-overrides) - (assoc-in [:webserver :ssl-port] port))] + (assoc-in [:webserver :ssl-port] port)) + config-file (ks/temp-file "puppetserver-conf" ".edn") + cmd ["java" "-cp" "puppetserver/target/puppet-server-release.jar" + "clojure.main" "-m" "puppetlabs.trapperkeeper.main" + "services" + "--bootstrap-config" dev-bootstrap-file + "-c" (.getPath config-file)] + adjust-env #(doto ^java.util.Map (.environment %) + (.remove "CLASSPATH")) + pb (doto (ProcessBuilder. cmd) + (.redirectOutput ProcessBuilder$Redirect/INHERIT) + (.redirectError ProcessBuilder$Redirect/INHERIT) + adjust-env) + _ (spit config-file (pr-str config)) + process (.start pb)] + (.addShutdownHook (Runtime/getRuntime) + (doto (Thread. #(.destroy process)) + (.setName (str "Subprocess cleanup for " process)))) + (log/info (str "Started puppetserver on port " port)) + (wait-for-server port tu/default-timeout-ms) (PuppetServerTestServer. {:hostname "localhost" :port port :code-dir "target/puppetserver/master-code" :conf-dir "target/puppetserver/master-conf"} - [(.getPath tmp-conf) + [(.getPath config-file) "target/puppetserver/master-conf" "target/puppetserver/master-code"] - (tkbs/bootstrap-services-with-config services config))))) + process)))) (defn run-puppet-server [pdb-servers config-overrides] (run-puppet-server-as "localhost" pdb-servers config-overrides)) From 7f094c6ed33ea25c5a43505f60d0c16f07fb80dc Mon Sep 17 00:00:00 2001 From: Rob Browning Date: Mon, 9 Jan 2023 17:38:57 -0600 Subject: [PATCH 17/27] (PDB-5572) Don't limit :integration test puppetserver clone depth This started breaking checkouts of say 6.x. --- ext/bin/config-puppetserver-test-ref | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/bin/config-puppetserver-test-ref b/ext/bin/config-puppetserver-test-ref index b0ef2e0841..dd41b491d5 100755 --- a/ext/bin/config-puppetserver-test-ref +++ b/ext/bin/config-puppetserver-test-ref @@ -63,7 +63,7 @@ if test -d puppetserver; then git -C puppetserver clean -fdx else # If repo doesn't exist, clone from GitHub - git clone --depth 10 -b "$pupsrv_ref" https://github.com/puppetlabs/puppetserver + git clone -b "$pupsrv_ref" https://github.com/puppetlabs/puppetserver fi (export PUPPETSERVER_HEAP_SIZE=1G From 2dff453b4df1b8903903ba1b360a1dfa3ace6d50 Mon Sep 17 00:00:00 2001 From: Jenkins CI Date: Wed, 2 Aug 2023 17:21:39 +0000 Subject: [PATCH 18/27] Set clj-parent=5.5.0 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 867111a5c4..0765707f9e 100644 --- a/project.clj +++ b/project.clj @@ -1,6 +1,6 @@ (def pdb-version "7.13.2-SNAPSHOT") -(def clj-parent-version "5.4.0") +(def clj-parent-version "5.5.0") (defn true-in-env? [x] (#{"true" "yes" "1"} (System/getenv x))) From ee95d0ec973a1c8d7953cd9151a42ec78d21ea78 Mon Sep 17 00:00:00 2001 From: rileynewton <95654936+rileynewton@users.noreply.github.com> Date: Wed, 2 Aug 2023 12:33:48 -0700 Subject: [PATCH 19/27] (PDB-5670, PDB-5668) Update ezbake and beaker-hostgenerator for rhel9 support --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 6b0686d00c..8a4a402784 100644 --- a/Gemfile +++ b/Gemfile @@ -64,7 +64,7 @@ if ENV['NO_ACCEPTANCE'] != 'true' gem 'beaker', '~> 4.1' end end - gem 'beaker-hostgenerator', '~> 1.12' + gem 'beaker-hostgenerator', '~> 2.2.3' gem 'beaker-abs', *location_for(ENV['BEAKER_ABS_VERSION'] || '~> 0.2') gem 'beaker-vmpooler', *location_for(ENV['BEAKER_VMPOOLER_VERSION'] || "~> 1.3") gem 'beaker-puppet', '~> 1.0' From ede5fdb3e78835222649a839473d269cd7ebacf1 Mon Sep 17 00:00:00 2001 From: Austin Blatt Date: Wed, 2 Aug 2023 13:36:27 -0700 Subject: [PATCH 20/27] (maint) exclude beaker gems from integration/rspec tests --- .github/workflows/main.yml | 1 + Gemfile | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 14f0219726..8f60417d49 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -89,6 +89,7 @@ jobs: LEIN_PROFILES: ${{ matrix.lein-profile }} PDB_QUERY_OPTIMIZE_DROP_UNUSED_JOINS: ${{ matrix.drop-joins }} PDB_USE_DEPRECATED_QUERY_STREAMING_METHOD: ${{ matrix.deprecated-query-streaming }} + NO_ACCEPTANCE: true run: ci/bin/prep-and-run-in github ${{ matrix.flavor }} - uses: actions/upload-artifact@v3 with: diff --git a/Gemfile b/Gemfile index 8a4a402784..0affb3a839 100644 --- a/Gemfile +++ b/Gemfile @@ -63,10 +63,10 @@ if ENV['NO_ACCEPTANCE'] != 'true' # use the pinned version gem 'beaker', '~> 4.1' end + gem 'beaker-hostgenerator', '~> 2.2.3' + gem 'beaker-abs', *location_for(ENV['BEAKER_ABS_VERSION'] || '~> 0.2') + gem 'beaker-vmpooler', *location_for(ENV['BEAKER_VMPOOLER_VERSION'] || "~> 1.3") + gem 'beaker-puppet', '~> 1.0' + gem 'faraday', '~> 1.8.0' end - gem 'beaker-hostgenerator', '~> 2.2.3' - gem 'beaker-abs', *location_for(ENV['BEAKER_ABS_VERSION'] || '~> 0.2') - gem 'beaker-vmpooler', *location_for(ENV['BEAKER_VMPOOLER_VERSION'] || "~> 1.3") - gem 'beaker-puppet', '~> 1.0' - gem 'faraday', '~> 1.8.0' end From 8b680f736e108505e4564f114c15c80f3454be11 Mon Sep 17 00:00:00 2001 From: Austin Blatt Date: Fri, 4 Aug 2023 08:23:19 -0700 Subject: [PATCH 21/27] (maint) Update ezbake to 2.5.3 contains a fix to allow package builds on el9 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 0765707f9e..16941a6732 100644 --- a/project.clj +++ b/project.clj @@ -313,7 +313,7 @@ ;; in the final package. [puppetlabs/puppetdb ~pdb-version :exclusions [com.zaxxer/HikariCP]]] :name "puppetdb" - :plugins [[puppetlabs/lein-ezbake "2.5.2"]]} + :plugins [[puppetlabs/lein-ezbake "2.5.3"]]} :testutils {:source-paths ^:replace ["test"] :resource-paths ^:replace [] ;; Something else may need adjustment, but From 0b916dca98ef0b04ccfd4d8d05653879cc27e02f Mon Sep 17 00:00:00 2001 From: Rob Browning Date: Mon, 7 Aug 2023 15:21:40 -0500 Subject: [PATCH 22/27] (PDB-5633) log-command-processed-message: handle nil producer-ts Don't fail to log just because the producer timestamp is nil. --- src/puppetlabs/puppetdb/command.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/puppetlabs/puppetdb/command.clj b/src/puppetlabs/puppetdb/command.clj index e9468f3c42..34da000b26 100644 --- a/src/puppetlabs/puppetdb/command.clj +++ b/src/puppetlabs/puppetdb/command.clj @@ -295,7 +295,8 @@ received-time (str (time/to-long received-time)) duration (str (in-millis (interval start-time (now)))) command-name (command-names command-kw) - producer-utc-s (-> producer-ts time/to-timestamp .getTime) + producer-utc-s (or (some-> producer-ts time/to-timestamp .getTime) + "nil") summary (str id "-" received-time "-" producer-utc-s) hash-prefix (if hash (let [hs (if (bytes? hash) From b7c692c16592e924dcc608998cbdadd38bcd4c77 Mon Sep 17 00:00:00 2001 From: Rob Browning Date: Mon, 7 Aug 2023 15:19:05 -0500 Subject: [PATCH 23/27] (PDB-5633) exec-command: handle nil getSQLState values Don't assume it won't be nil; sometimes it is. --- src/puppetlabs/puppetdb/command.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/puppetlabs/puppetdb/command.clj b/src/puppetlabs/puppetdb/command.clj index 34da000b26..15cbaa0a3a 100644 --- a/src/puppetlabs/puppetdb/command.clj +++ b/src/puppetlabs/puppetdb/command.clj @@ -529,11 +529,11 @@ ;; also assuming that all commands don't vary their behavior ;; across retries sufficiently for that to matter either. ;; This was originally introduced to handle - ;; program_limit_exceeded errors caused by attemptps to insert + ;; program_limit_exceeded errors caused by attempts to insert ;; resource titles that were too big to fit in a postgres ;; index. ;; cf. https://www.postgresql.org/docs/11/errcodes-appendix.html - (when (str/starts-with? (.getSQLState ex) "54") + (when (some-> ex .getSQLState (str/starts-with? "54")) (throw (fatality ex))) (throw ex))))) From 911f88f500ef3597ff93950dd8e4a9de0db74528 Mon Sep 17 00:00:00 2001 From: Steve Axthelm Date: Wed, 16 Aug 2023 09:31:21 -0700 Subject: [PATCH 24/27] (PDB-5667, PDB-5671) add el-9 to supported platforms --- acceptance/helper.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/acceptance/helper.rb b/acceptance/helper.rb index 286b4cd7d9..6bca54157c 100644 --- a/acceptance/helper.rb +++ b/acceptance/helper.rb @@ -301,6 +301,10 @@ def is_el8() test_config[:os_families].has_key?('centos8-64-1') end + def is_el9() + return test_config[:os_families].has_key?('redhat9-64-1') + end + def is_rhel7fips return test_config[:os_families].has_key? 'redhatfips7-64-1' end @@ -348,6 +352,8 @@ def oldest_supported '7.9.0' elsif is_el8 '6.0.3' + elsif is_el9 + '7.13.2' elsif is_rhel7fips '6.4.0' elsif is_buster @@ -417,6 +423,8 @@ def get_package_version(host, version = nil) "#{version}.el7" elsif host['platform'].include?('el-8') "#{version}.el8" + elsif host['platform'].include?('el-9') + "#{version}.el9" elsif host['platform'].include?('ubuntu-18.04') "#{version}bionic" elsif host['platform'].include?('ubuntu-20.04') From d4ea05e7b596f5c26407d8df0151b017b290fae3 Mon Sep 17 00:00:00 2001 From: Steve Axthelm Date: Wed, 16 Aug 2023 15:53:10 -0700 Subject: [PATCH 25/27] (PDB-5675) Add 7.14.0 release notes --- documentation/release_notes_7.markdown | 27 ++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/documentation/release_notes_7.markdown b/documentation/release_notes_7.markdown index 97266e5b44..5489dd5232 100644 --- a/documentation/release_notes_7.markdown +++ b/documentation/release_notes_7.markdown @@ -15,6 +15,25 @@ canonical: "/puppetdb/latest/release_notes.html" # PuppetDB: Release notes +## PuppetDB 7.14.0 + +Released August 22 2023 + +## New features and improvements + +* Add el-9 as supported platform + [PDB-5671](https://perforce.atlassian.net/browse/PDB-5671) + +* PQL parsing of OR clauses can result in OOM errors + [PDB-5643](https://perforce.atlassian.net/browse/PDB-5643) + +* Add ubuntu 2204 as supported platform + [PDB-5636](https://perforce.atlassian.net/browse/PDB-5636) + +### Contributors + +Austin Blatt, Nick Burgan-Illig, Jonathan Newman, Eric Newton, Joshua Partlow, Steve Axthelm, and Rob Browning + ## PuppetDB 7.13.1 Released June 14 2023 @@ -37,16 +56,16 @@ Released April 6 2023 ### Bug fixes -* queries with thousands of `in array` entries would cause performance problems +* queries with thousands of `in array` entries would cause performance problems in query compilation. ([PDB-3171](https://tickets.puppetlabs.com/browse/PDB-3171)) -* PuppetDB should no longer crash on reload (SIGHUP) in some cases +* PuppetDB should no longer crash on reload (SIGHUP) in some cases (e.g. after startup but before processing any commands). ([PDB-5215](https://tickets.puppetlabs.com/browse/PDB-5215)) ## New features and improvements -* PuppetDB installations with PostgreSQL 14+ will detach reports and resource_events +* PuppetDB installations with PostgreSQL 14+ will detach reports and resource_events partitions concurrently before dropping them. ([PDB-5554](https://tickets.puppetlabs.com/browse/PDB-5554)) * The reports and resource_events tables were migrated to use PostgreSQL declarative partitioning in support of PDB-5554. @@ -329,7 +348,7 @@ Austin Blatt, Bogdan Irimie, Rob Browning, Sebastian Miclea, and Stel Abrego ### Bug fixes * If a query with an extract clause contains a misspelled option, the clause is completely ignored resulting in a misleading response body. - ``` + ``` ["from", "reports", ["extract", [["function", "count", "certname"]], ["null?", "type", false], From 064a4de6c5aac4b13e184011d3cdf1d872a9520f Mon Sep 17 00:00:00 2001 From: Steve Axthelm Date: Thu, 17 Aug 2023 14:47:54 -0700 Subject: [PATCH 26/27] (maint) Update version to 7.14.0 for release --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 16941a6732..03dae42f55 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(def pdb-version "7.13.2-SNAPSHOT") +(def pdb-version "7.14.0") (def clj-parent-version "5.5.0") From f5c89600d20ee7257e82fecbcec98fcde8676cd2 Mon Sep 17 00:00:00 2001 From: Steve Axthelm Date: Thu, 17 Aug 2023 14:47:54 -0700 Subject: [PATCH 27/27] (maint) Update version to 7.14.1-SNAPSHOT --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 03dae42f55..ddb0b08933 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(def pdb-version "7.14.0") +(def pdb-version "7.14.1-SNAPSHOT") (def clj-parent-version "5.5.0")