diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 14f0219726..8f60417d49 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -89,6 +89,7 @@ jobs: LEIN_PROFILES: ${{ matrix.lein-profile }} PDB_QUERY_OPTIMIZE_DROP_UNUSED_JOINS: ${{ matrix.drop-joins }} PDB_USE_DEPRECATED_QUERY_STREAMING_METHOD: ${{ matrix.deprecated-query-streaming }} + NO_ACCEPTANCE: true run: ci/bin/prep-and-run-in github ${{ matrix.flavor }} - uses: actions/upload-artifact@v3 with: diff --git a/Gemfile b/Gemfile index 6b0686d00c..0affb3a839 100644 --- a/Gemfile +++ b/Gemfile @@ -63,10 +63,10 @@ if ENV['NO_ACCEPTANCE'] != 'true' # use the pinned version gem 'beaker', '~> 4.1' end + gem 'beaker-hostgenerator', '~> 2.2.3' + gem 'beaker-abs', *location_for(ENV['BEAKER_ABS_VERSION'] || '~> 0.2') + gem 'beaker-vmpooler', *location_for(ENV['BEAKER_VMPOOLER_VERSION'] || "~> 1.3") + gem 'beaker-puppet', '~> 1.0' + gem 'faraday', '~> 1.8.0' end - gem 'beaker-hostgenerator', '~> 1.12' - gem 'beaker-abs', *location_for(ENV['BEAKER_ABS_VERSION'] || '~> 0.2') - gem 'beaker-vmpooler', *location_for(ENV['BEAKER_VMPOOLER_VERSION'] || "~> 1.3") - gem 'beaker-puppet', '~> 1.0' - gem 'faraday', '~> 1.8.0' end diff --git a/acceptance/helper.rb b/acceptance/helper.rb index 131121ba6c..6bca54157c 100644 --- a/acceptance/helper.rb +++ b/acceptance/helper.rb @@ -284,6 +284,10 @@ def is_focal() return test_config[:os_families].has_key? 'ubuntu2004-64-1' end + def is_jammy() + return test_config[:os_families].has_key? 'ubuntu2204-64-1' + end + def is_buster() return test_config[:os_families].has_key? 'debian10-64-1' end @@ -297,6 +301,10 @@ def is_el8() test_config[:os_families].has_key?('centos8-64-1') end + def is_el9() + return test_config[:os_families].has_key?('redhat9-64-1') + end + def is_rhel7fips return test_config[:os_families].has_key? 'redhatfips7-64-1' end @@ -339,17 +347,21 @@ def puppet_repo_version(platform_version, install_type, nightly) # supported version of PuppetDB. Its version must be available in the # platform version returned by puppet_repo_version above def oldest_supported - # account for bionic/rhel8 not having build before certian versions + # account for bionic/rhel8 not having build before certain versions if is_bullseye '7.9.0' elsif is_el8 '6.0.3' + elsif is_el9 + '7.13.2' elsif is_rhel7fips '6.4.0' elsif is_buster '6.7.0' elsif is_focal '6.12.0' + elsif is_jammy + '7.13.2' else '6.0.0' end @@ -411,10 +423,14 @@ def get_package_version(host, version = nil) "#{version}.el7" elsif host['platform'].include?('el-8') "#{version}.el8" + elsif host['platform'].include?('el-9') + "#{version}.el9" elsif host['platform'].include?('ubuntu-18.04') "#{version}bionic" elsif host['platform'].include?('ubuntu-20.04') "#{version}focal" + elsif host['platform'].include?('ubuntu-22.04') + "#{version}jammy" elsif host['platform'].include?('debian-10') "#{version}buster" elsif host['platform'].include?('debian-11') diff --git a/documentation/release_notes_7.markdown b/documentation/release_notes_7.markdown index 97266e5b44..5489dd5232 100644 --- a/documentation/release_notes_7.markdown +++ b/documentation/release_notes_7.markdown @@ -15,6 +15,25 @@ canonical: "/puppetdb/latest/release_notes.html" # PuppetDB: Release notes +## PuppetDB 7.14.0 + +Released August 22 2023 + +## New features and improvements + +* Add el-9 as supported platform + [PDB-5671](https://perforce.atlassian.net/browse/PDB-5671) + +* PQL parsing of OR clauses can result in OOM errors + [PDB-5643](https://perforce.atlassian.net/browse/PDB-5643) + +* Add ubuntu 2204 as supported platform + [PDB-5636](https://perforce.atlassian.net/browse/PDB-5636) + +### Contributors + +Austin Blatt, Nick Burgan-Illig, Jonathan Newman, Eric Newton, Joshua Partlow, Steve Axthelm, and Rob Browning + ## PuppetDB 7.13.1 Released June 14 2023 @@ -37,16 +56,16 @@ Released April 6 2023 ### Bug fixes -* queries with thousands of `in array` entries would cause performance problems +* queries with thousands of `in array` entries would cause performance problems in query compilation. ([PDB-3171](https://tickets.puppetlabs.com/browse/PDB-3171)) -* PuppetDB should no longer crash on reload (SIGHUP) in some cases +* PuppetDB should no longer crash on reload (SIGHUP) in some cases (e.g. after startup but before processing any commands). ([PDB-5215](https://tickets.puppetlabs.com/browse/PDB-5215)) ## New features and improvements -* PuppetDB installations with PostgreSQL 14+ will detach reports and resource_events +* PuppetDB installations with PostgreSQL 14+ will detach reports and resource_events partitions concurrently before dropping them. ([PDB-5554](https://tickets.puppetlabs.com/browse/PDB-5554)) * The reports and resource_events tables were migrated to use PostgreSQL declarative partitioning in support of PDB-5554. @@ -329,7 +348,7 @@ Austin Blatt, Bogdan Irimie, Rob Browning, Sebastian Miclea, and Stel Abrego ### Bug fixes * If a query with an extract clause contains a misspelled option, the clause is completely ignored resulting in a misleading response body. - ``` + ``` ["from", "reports", ["extract", [["function", "count", "certname"]], ["null?", "type", false], diff --git a/ext/bin/config-puppet-test-ref b/ext/bin/config-puppet-test-ref index cf314abda6..51d56ee894 100755 --- a/ext/bin/config-puppet-test-ref +++ b/ext/bin/config-puppet-test-ref @@ -49,7 +49,9 @@ case "$#" in esac # Install and update Ruby dependencies from top-level Gemfile -bundle install --without acceptance --path vendor/bundle --retry=10 +bundle config set --local path vendor/bundle +bundle config set --local without acceptance +bundle install --retry=10 bundle update # Print out info on Puppet dependency bundle info puppet || true # Not all versions appear to have info @@ -59,6 +61,6 @@ bundle info puppet || true # Not all versions appear to have info cd vendor # This will something like .../vendor/bundle/ruby/2.7.0/bundler/gems/puppet-ae5379e03311 # It is a Puppet git repository installed by bundler -puppet_path="$(bundle show puppet)" +puppet_path="$(bundle info puppet --path)" # Create a symlink at `vendor/puppet` that points to Puppet repository "$top/ext/bin/symlink-relative-to" "$puppet_path" puppet diff --git a/ext/bin/config-puppetserver-test-ref b/ext/bin/config-puppetserver-test-ref index e86064881b..dd41b491d5 100755 --- a/ext/bin/config-puppetserver-test-ref +++ b/ext/bin/config-puppetserver-test-ref @@ -63,7 +63,7 @@ if test -d puppetserver; then git -C puppetserver clean -fdx else # If repo doesn't exist, clone from GitHub - git clone --depth 10 -b "$pupsrv_ref" https://github.com/puppetlabs/puppetserver + git clone -b "$pupsrv_ref" https://github.com/puppetlabs/puppetserver fi (export PUPPETSERVER_HEAP_SIZE=1G @@ -72,6 +72,7 @@ fi # If we're not testing against the git tree (say in jenkins), don't install. if test -z "$PDB_NO_PUPPETSERVER_INSTALL"; then lein install + lein uberjar fi # Cache Puppet Server version in a file in ext/test-conf directory dep_ver="$(lein-pprint :version)" diff --git a/locales/puppetdb.pot b/locales/puppetdb.pot index 29b13ad4ad..09196ef6a6 100644 --- a/locales/puppetdb.pot +++ b/locales/puppetdb.pot @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: puppetlabs.puppetdb \n" -"X-Git-Ref: 1a530198597b03481c6b50fa1e4977d7dcd82bea\n" +"X-Git-Ref: f768ede7ed7251d1a06dd027462d2ec9e3392ccd\n" "Report-Msgid-Bugs-To: docs@puppet.com\n" "POT-Creation-Date: \n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" @@ -1593,7 +1593,8 @@ msgid "Creating additional index `catalog_resources_file_trgm`" msgstr "" #: src/puppetlabs/puppetdb/scf/migrate.clj -msgid "Disconnecting all {0} connections to {1} database before migrating" +msgid "" +"Disconnecting all non-migrator connections to {0} database before migrating" msgstr "" #: src/puppetlabs/puppetdb/scf/migrate.clj diff --git a/project.clj b/project.clj index 0a775a97c2..ddb0b08933 100644 --- a/project.clj +++ b/project.clj @@ -1,6 +1,6 @@ -(def pdb-version "7.13.2-SNAPSHOT") +(def pdb-version "7.14.1-SNAPSHOT") -(def clj-parent-version "5.3.7") +(def clj-parent-version "5.5.0") (defn true-in-env? [x] (#{"true" "yes" "1"} (System/getenv x))) @@ -79,6 +79,7 @@ :exclusions [com.andrewmcveigh/cljs-time]] [riddley "0.1.12"] [io.forward/yaml "1.0.5"] + [clj-commons/clj-yaml "1.0.26"] ;; Only needed for :integration tests [puppetlabs/trapperkeeper-filesystem-watcher nil]] @@ -312,7 +313,7 @@ ;; in the final package. [puppetlabs/puppetdb ~pdb-version :exclusions [com.zaxxer/HikariCP]]] :name "puppetdb" - :plugins [[puppetlabs/lein-ezbake "2.4.1"]]} + :plugins [[puppetlabs/lein-ezbake "2.5.3"]]} :testutils {:source-paths ^:replace ["test"] :resource-paths ^:replace [] ;; Something else may need adjustment, but diff --git a/resources/puppetlabs/puppetdb/pql/pql-grammar.ebnf b/resources/puppetlabs/puppetdb/pql/pql-grammar.ebnf index da350c57e7..d65aff7fc0 100644 --- a/resources/puppetlabs/puppetdb/pql/pql-grammar.ebnf +++ b/resources/puppetlabs/puppetdb/pql/pql-grammar.ebnf @@ -52,8 +52,8 @@ extract = , [], [extractfields], [], = expr-or; -expr-or = expr-and { , or, , expr-or }; -expr-and = expr-not { , and, , expr-and }; +expr-or = expr-and | expr-and, , or, , expr-or; +expr-and = expr-not | expr-not, , and, , expr-and; expr-not = ( not, [], expr-not ) | expr-rest; = ( , [], expression, [], ) | condexpression | condexpnull | subquery; diff --git a/src/puppetlabs/puppetdb/command.clj b/src/puppetlabs/puppetdb/command.clj index e9468f3c42..15cbaa0a3a 100644 --- a/src/puppetlabs/puppetdb/command.clj +++ b/src/puppetlabs/puppetdb/command.clj @@ -295,7 +295,8 @@ received-time (str (time/to-long received-time)) duration (str (in-millis (interval start-time (now)))) command-name (command-names command-kw) - producer-utc-s (-> producer-ts time/to-timestamp .getTime) + producer-utc-s (or (some-> producer-ts time/to-timestamp .getTime) + "nil") summary (str id "-" received-time "-" producer-utc-s) hash-prefix (if hash (let [hs (if (bytes? hash) @@ -528,11 +529,11 @@ ;; also assuming that all commands don't vary their behavior ;; across retries sufficiently for that to matter either. ;; This was originally introduced to handle - ;; program_limit_exceeded errors caused by attemptps to insert + ;; program_limit_exceeded errors caused by attempts to insert ;; resource titles that were too big to fit in a postgres ;; index. ;; cf. https://www.postgresql.org/docs/11/errcodes-appendix.html - (when (str/starts-with? (.getSQLState ex) "54") + (when (some-> ex .getSQLState (str/starts-with? "54")) (throw (fatality ex))) (throw ex))))) diff --git a/src/puppetlabs/puppetdb/pql/transform.clj b/src/puppetlabs/puppetdb/pql/transform.clj index ef321471e7..797f146577 100644 --- a/src/puppetlabs/puppetdb/pql/transform.clj +++ b/src/puppetlabs/puppetdb/pql/transform.clj @@ -33,13 +33,25 @@ ;; Single arg? collapse ([data] data) ;; Multiple args? turn it into an or statement - ([data & args] (vec (concat ["or" data] args)))) + ([data & args] + (->> (cons data args) + (mapcat #(if (and (vector? %) (= (first %) "or")) + (rest %) + [%])) + (cons "or") + (vec)))) (defn transform-expr-and ;; Single arg? collapse ([data] data) ;; Multiple args? turn it into an and statement - ([data & args] (vec (concat ["and" data] args)))) + ([data & args] + (->> (cons data args) + (mapcat #(if (and (vector? %) (= (first %) "and")) + (rest %) + [%])) + (cons "and") + (vec)))) (defn transform-expr-not ;; Single arg? Just collapse the :expr-not and pass back the data, diff --git a/src/puppetlabs/puppetdb/scf/migrate.clj b/src/puppetlabs/puppetdb/scf/migrate.clj index 411e55efba..7f0a9dafcb 100644 --- a/src/puppetlabs/puppetdb/scf/migrate.clj +++ b/src/puppetlabs/puppetdb/scf/migrate.clj @@ -2555,8 +2555,8 @@ (if-not user (do (log/info - (trs "Disconnecting all {0} connections to {1} database before migrating" - user db-name)) + (trs "Disconnecting all non-migrator connections to {0} database before migrating" + db-name)) (doseq [user users] ;; Because the revoke may not actually produce an error when ;; it doesn't work. @@ -2617,16 +2617,17 @@ tables))] (if-not write-user (migrate) - (call-with-connections-blocked-during-migration - db-name - (distinct [read-user write-user]) - (fn [] - ;; So new tables, etc. are owned by the write-user - (jdbc/do-commands (str "set role " (jdbc/double-quote write-user))) - (try! - (migrate) - (finally - (jdbc/do-commands (str "set role " (jdbc/double-quote orig-user)))))))))) + (when-not (empty? (pending-migrations)) + (call-with-connections-blocked-during-migration + db-name + (distinct [read-user write-user]) + (fn [] + ;; So new tables, etc. are owned by the write-user + (jdbc/do-commands (str "set role " (jdbc/double-quote write-user))) + (try! + (migrate) + (finally + (jdbc/do-commands (str "set role " (jdbc/double-quote orig-user))))))))))) (defn initialize-schema "Ensures the database is migrated to the latest version, and returns diff --git a/test/puppetlabs/puppetdb/integration/fixtures.clj b/test/puppetlabs/puppetdb/integration/fixtures.clj index 3eb449f848..c4affe51b4 100644 --- a/test/puppetlabs/puppetdb/integration/fixtures.clj +++ b/test/puppetlabs/puppetdb/integration/fixtures.clj @@ -13,13 +13,14 @@ [puppetlabs.puppetdb.testutils.db :as dbutils] [puppetlabs.puppetdb.testutils.services :as svc-utils] [puppetlabs.trapperkeeper.app :as tk-app] - [puppetlabs.trapperkeeper.bootstrap :as tk-bootstrap] [puppetlabs.trapperkeeper.config :as tk-config] [puppetlabs.trapperkeeper.testutils.bootstrap :as tkbs] [yaml.core :as yaml] [puppetlabs.puppetdb.time :as time] [puppetlabs.puppetdb.utils :as utils]) - (:import [com.typesafe.config ConfigValueFactory])) + (:import + (com.typesafe.config ConfigValueFactory) + (java.lang ProcessBuilder ProcessBuilder$Redirect))) (defprotocol TestServer (server-info [this])) @@ -154,13 +155,13 @@ ;;; Puppet Server fixture -(defrecord PuppetServerTestServer [info-map files-to-cleanup app] +(defrecord PuppetServerTestServer [info-map files-to-cleanup process] TestServer (server-info [_] info-map) java.lang.AutoCloseable (close [_] - (tk-app/stop app) + (doto process .destroy .waitFor) ;; Assumes destroy sends a SIGTERM (doseq [f files-to-cleanup] (fs/delete f)))) (def dev-config-file "./test-resources/puppetserver/puppetserver.conf") @@ -201,6 +202,21 @@ (clojure.string/join ","))}} (or overrides {}))))) +(defn- wait-for-server + [port timeout-ms] + (loop [n (/ timeout-ms 100)] + (when (neg? n) + (throw (ex-info (str "server not ready within " timeout-ms "ms") {}))) + (let [res (try + (svc-utils/get-ssl (str "https://localhost:" port "/status/v1/services")) + (catch java.net.ConnectException _ + ::nope))] + (cond + (= res ::nope) (do (Thread/sleep 100) (recur (dec n))) + (= 200 (:status res)) true + :else (throw (ex-info "Unexpected result from status endpoint while waiting" + res)))))) + (defn run-puppet-server-as [node-name pdb-servers config-overrides] (let [puppetdb-conf (io/file "target/puppetserver/master-conf/puppetdb.conf") puppet-conf (io/file "target/puppetserver/master-conf/puppet.conf") @@ -224,21 +240,37 @@ (write-puppetdb-terminus-config pdb-servers puppetdb-conf terminus-config-overrides) - (let [services (tk-bootstrap/parse-bootstrap-config! dev-bootstrap-file) - tmp-conf (ks/temp-file "puppetserver" ".conf") - _ (fs/copy dev-config-file tmp-conf) - port (svc-utils/open-port-num) - config (-> (tk-config/load-config (.getPath tmp-conf)) + (let [port (svc-utils/open-port-num) + config (-> (tk-config/load-config dev-config-file) (merge puppetserver-config-overrides) - (assoc-in [:webserver :ssl-port] port))] + (assoc-in [:webserver :ssl-port] port)) + config-file (ks/temp-file "puppetserver-conf" ".edn") + cmd ["java" "-cp" "puppetserver/target/puppet-server-release.jar" + "clojure.main" "-m" "puppetlabs.trapperkeeper.main" + "services" + "--bootstrap-config" dev-bootstrap-file + "-c" (.getPath config-file)] + adjust-env #(doto ^java.util.Map (.environment %) + (.remove "CLASSPATH")) + pb (doto (ProcessBuilder. cmd) + (.redirectOutput ProcessBuilder$Redirect/INHERIT) + (.redirectError ProcessBuilder$Redirect/INHERIT) + adjust-env) + _ (spit config-file (pr-str config)) + process (.start pb)] + (.addShutdownHook (Runtime/getRuntime) + (doto (Thread. #(.destroy process)) + (.setName (str "Subprocess cleanup for " process)))) + (log/info (str "Started puppetserver on port " port)) + (wait-for-server port tu/default-timeout-ms) (PuppetServerTestServer. {:hostname "localhost" :port port :code-dir "target/puppetserver/master-code" :conf-dir "target/puppetserver/master-conf"} - [(.getPath tmp-conf) + [(.getPath config-file) "target/puppetserver/master-conf" "target/puppetserver/master-code"] - (tkbs/bootstrap-services-with-config services config))))) + process)))) (defn run-puppet-server [pdb-servers config-overrides] (run-puppet-server-as "localhost" pdb-servers config-overrides)) diff --git a/test/puppetlabs/puppetdb/pql/parser_test.clj b/test/puppetlabs/puppetdb/pql/parser_test.clj index 134f3a8e95..b1b8e8139f 100644 --- a/test/puppetlabs/puppetdb/pql/parser_test.clj +++ b/test/puppetlabs/puppetdb/pql/parser_test.clj @@ -285,11 +285,11 @@ [:condexpression [:field "d"] "=" [:integer "4"]]] [:expr-and [:expr-not - [:condexpression [:field "a"] "=" [:integer "1"]]]]]] + [:condexpression [:field "a"] "=" [:integer "1"]]]]] [:expr-or [:expr-and [:expr-not - [:condexpression [:field "b"] "=" [:integer "2"]]]]]]] + [:condexpression [:field "b"] "=" [:integer "2"]]]]]]]] "(c = 3 or d = 4) and (a = 1 or b = 2)" [[:expr-or @@ -324,6 +324,38 @@ "a=1 b=2" "")) +;; PDB-5643 Check that a PQL expression with many OR clauses parses in reasonable time. +(deftest test-expression-depth-performance + (let [expression "(certname = 'foo22' or + certname = 'foo21' or + certname = 'foo20' or + certname = 'foo19' or + certname = 'foo18' or + certname = 'foo17' or + certname = 'foo16' or + certname = 'foo15' or + certname = 'foo14' or + certname = 'foo13' or + certname = 'foo12' or + certname = 'foo11' or + certname = 'foo10' or + certname = 'foo9' or + certname = 'foo8' or + certname = 'foo7' or + certname = 'foo6' or + certname = 'foo5' or + certname = 'foo4' or + certname = 'foo3' or + certname = 'foo2' or + certname = 'foo1')" + start-time (System/currentTimeMillis) + expected-duration 500] + (parse expression :start :expression) + (let [duration-millis (- (System/currentTimeMillis) start-time)] + (is + (< duration-millis expected-duration) + (format "(PDB-5643) Parsing a PQL expression with many OR clauses took %s milliseconds, which is longer than the alloted %s" duration-millis expected-duration))))) + (deftest test-subquery (are [in expected] (= (parse in :start :subquery) expected) "nodes{}" diff --git a/test/puppetlabs/puppetdb/pql_test.clj b/test/puppetlabs/puppetdb/pql_test.clj index 99fc0554b9..11452c5af5 100644 --- a/test/puppetlabs/puppetdb/pql_test.clj +++ b/test/puppetlabs/puppetdb/pql_test.clj @@ -3,8 +3,8 @@ [puppetlabs.puppetdb.pql :as pql])) (deftest test-pql->ast - (are [pql ast] (= (first (pql/pql->ast pql)) - ast) + (are [pql ast] (= ast + (first (pql/pql->ast pql))) ;; Some basic comparisons