Merge pull request #3865 from steveax/doc-7.y

Merge remote-tracking branch 'puppet/7.x' into doc-7.y

.github/workflows/main.yml

@@ -89,6 +89,7 @@ jobs:
  LEIN_PROFILES: ${{ matrix.lein-profile }}
  PDB_QUERY_OPTIMIZE_DROP_UNUSED_JOINS: ${{ matrix.drop-joins }}
  PDB_USE_DEPRECATED_QUERY_STREAMING_METHOD: ${{ matrix.deprecated-query-streaming }}
+ NO_ACCEPTANCE: true
  run: ci/bin/prep-and-run-in github ${{ matrix.flavor }}
  - uses: actions/upload-artifact@v3
  with:

Gemfile

@@ -63,10 +63,10 @@ if ENV['NO_ACCEPTANCE'] != 'true'
  # use the pinned version
  gem 'beaker', '~> 4.1'
  end
+ gem 'beaker-hostgenerator', '~> 2.2.3'
+ gem 'beaker-abs', *location_for(ENV['BEAKER_ABS_VERSION'] || '~> 0.2')
+ gem 'beaker-vmpooler', *location_for(ENV['BEAKER_VMPOOLER_VERSION'] || "~> 1.3")
+ gem 'beaker-puppet', '~> 1.0'
+ gem 'faraday', '~> 1.8.0'
  end
- gem 'beaker-hostgenerator', '~> 1.12'
- gem 'beaker-abs', *location_for(ENV['BEAKER_ABS_VERSION'] || '~> 0.2')
- gem 'beaker-vmpooler', *location_for(ENV['BEAKER_VMPOOLER_VERSION'] || "~> 1.3")
- gem 'beaker-puppet', '~> 1.0'
- gem 'faraday', '~> 1.8.0'
 end

acceptance/helper.rb

@@ -284,6 +284,10 @@ def is_focal()
  return test_config[:os_families].has_key? 'ubuntu2004-64-1'
  end
 
+ def is_jammy()
+ return test_config[:os_families].has_key? 'ubuntu2204-64-1'
+ end
+
  def is_buster()
  return test_config[:os_families].has_key? 'debian10-64-1'
  end
@@ -297,6 +301,10 @@ def is_el8()
  test_config[:os_families].has_key?('centos8-64-1')
  end
 
+ def is_el9()
+ return test_config[:os_families].has_key?('redhat9-64-1')
+ end
+
  def is_rhel7fips
  return test_config[:os_families].has_key? 'redhatfips7-64-1'
  end
@@ -339,17 +347,21 @@ def puppet_repo_version(platform_version, install_type, nightly)
  # supported version of PuppetDB. Its version must be available in the
  # platform version returned by puppet_repo_version above
  def oldest_supported
- # account for bionic/rhel8 not having build before certian versions
+ # account for bionic/rhel8 not having build before certain versions
  if is_bullseye
  '7.9.0'
  elsif is_el8
  '6.0.3'
+ elsif is_el9
+ '7.13.2'
  elsif is_rhel7fips
  '6.4.0'
  elsif is_buster
  '6.7.0'
  elsif is_focal
  '6.12.0'
+ elsif is_jammy
+ '7.13.2'
  else
  '6.0.0'
  end
@@ -411,10 +423,14 @@ def get_package_version(host, version = nil)
  "#{version}.el7"
  elsif host['platform'].include?('el-8')
  "#{version}.el8"
+ elsif host['platform'].include?('el-9')
+ "#{version}.el9"
  elsif host['platform'].include?('ubuntu-18.04')
  "#{version}bionic"
  elsif host['platform'].include?('ubuntu-20.04')
  "#{version}focal"
+ elsif host['platform'].include?('ubuntu-22.04')
+ "#{version}jammy"
  elsif host['platform'].include?('debian-10')
  "#{version}buster"
  elsif host['platform'].include?('debian-11')

documentation/release_notes_7.markdown

@@ -15,6 +15,25 @@ canonical: "/puppetdb/latest/release_notes.html"
 
 # PuppetDB: Release notes
 
+## PuppetDB 7.14.0
+
+Released August 22 2023
+
+## New features and improvements
+
+* Add el-9 as supported platform
+ [PDB-5671](https://perforce.atlassian.net/browse/PDB-5671)
+
+* PQL parsing of OR clauses can result in OOM errors
+ [PDB-5643](https://perforce.atlassian.net/browse/PDB-5643)
+
+* Add ubuntu 2204 as supported platform
+ [PDB-5636](https://perforce.atlassian.net/browse/PDB-5636)
+
+### Contributors
+
+Austin Blatt, Nick Burgan-Illig, Jonathan Newman, Eric Newton, Joshua Partlow, Steve Axthelm, and Rob Browning
+
 ## PuppetDB 7.13.1
 
 Released June 14 2023
@@ -37,16 +56,16 @@ Released April 6 2023
 
 ### Bug fixes
 
-* queries with thousands of `in array` entries would cause performance problems 
+* queries with thousands of `in array` entries would cause performance problems
  in query compilation.
  ([PDB-3171](https://tickets.puppetlabs.com/browse/PDB-3171))
-* PuppetDB should no longer crash on reload (SIGHUP) in some cases 
+* PuppetDB should no longer crash on reload (SIGHUP) in some cases
  (e.g. after startup but before processing any commands).
  ([PDB-5215](https://tickets.puppetlabs.com/browse/PDB-5215))
 
 ## New features and improvements
 
-* PuppetDB installations with PostgreSQL 14+ will detach reports and resource_events 
+* PuppetDB installations with PostgreSQL 14+ will detach reports and resource_events
  partitions concurrently before dropping them.
  ([PDB-5554](https://tickets.puppetlabs.com/browse/PDB-5554))
 * The reports and resource_events tables were migrated to use PostgreSQL declarative partitioning in support of PDB-5554.
@@ -329,7 +348,7 @@ Austin Blatt, Bogdan Irimie, Rob Browning, Sebastian Miclea, and Stel Abrego
 ### Bug fixes
 
 * If a query with an extract clause contains a misspelled option, the clause is completely ignored resulting in a misleading response body.
- ``` 
+ ```
  ["from", "reports",
  ["extract", [["function", "count", "certname"]],
  ["null?", "type", false],

ext/bin/config-puppet-test-ref

@@ -49,7 +49,9 @@ case "$#" in
 esac
 
 # Install and update Ruby dependencies from top-level Gemfile
-bundle install --without acceptance --path vendor/bundle --retry=10
+bundle config set --local path vendor/bundle
+bundle config set --local without acceptance
+bundle install --retry=10
 bundle update
 # Print out info on Puppet dependency
 bundle info puppet || true # Not all versions appear to have info
@@ -59,6 +61,6 @@ bundle info puppet || true # Not all versions appear to have info
 cd vendor
 # This will something like .../vendor/bundle/ruby/2.7.0/bundler/gems/puppet-ae5379e03311
 # It is a Puppet git repository installed by bundler
-puppet_path="$(bundle show puppet)"
+puppet_path="$(bundle info puppet --path)"
 # Create a symlink at `vendor/puppet` that points to Puppet repository
 "$top/ext/bin/symlink-relative-to" "$puppet_path" puppet

ext/bin/config-puppetserver-test-ref

@@ -63,7 +63,7 @@ if test -d puppetserver; then
  git -C puppetserver clean -fdx
 else
  # If repo doesn't exist, clone from GitHub
- git clone --depth 10 -b "$pupsrv_ref" https://github.com/puppetlabs/puppetserver
+ git clone -b "$pupsrv_ref" https://github.com/puppetlabs/puppetserver
 fi
 
 (export PUPPETSERVER_HEAP_SIZE=1G
@@ -72,6 +72,7 @@ fi
  # If we're not testing against the git tree (say in jenkins), don't install.
  if test -z "$PDB_NO_PUPPETSERVER_INSTALL"; then
  lein install
+ lein uberjar
  fi
  # Cache Puppet Server version in a file in ext/test-conf directory
  dep_ver="$(lein-pprint :version)"

locales/puppetdb.pot

@@ -7,7 +7,7 @@
 msgid ""
 msgstr ""
 "Project-Id-Version: puppetlabs.puppetdb \n"
-"X-Git-Ref: 1a530198597b03481c6b50fa1e4977d7dcd82bea\n"
+"X-Git-Ref: f768ede7ed7251d1a06dd027462d2ec9e3392ccd\n"
 "Report-Msgid-Bugs-To: docs@puppet.com\n"
 "POT-Creation-Date: \n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
@@ -1593,7 +1593,8 @@ msgid "Creating additional index `catalog_resources_file_trgm`"
 msgstr ""
 
 #: src/puppetlabs/puppetdb/scf/migrate.clj
-msgid "Disconnecting all {0} connections to {1} database before migrating"
+msgid ""
+"Disconnecting all non-migrator connections to {0} database before migrating"
 msgstr ""
 
 #: src/puppetlabs/puppetdb/scf/migrate.clj

project.clj

@@ -1,6 +1,6 @@
-(def pdb-version "7.13.2-SNAPSHOT")
+(def pdb-version "7.14.1-SNAPSHOT")
 
-(def clj-parent-version "5.3.7")
+(def clj-parent-version "5.5.0")
 
 (defn true-in-env? [x]
  (#{"true" "yes" "1"} (System/getenv x)))
@@ -79,6 +79,7 @@
  :exclusions [com.andrewmcveigh/cljs-time]]
  [riddley "0.1.12"]
  [io.forward/yaml "1.0.5"]
+ [clj-commons/clj-yaml "1.0.26"]
 
  ;; Only needed for :integration tests
  [puppetlabs/trapperkeeper-filesystem-watcher nil]]
@@ -312,7 +313,7 @@
  ;; in the final package.
  [puppetlabs/puppetdb ~pdb-version :exclusions [com.zaxxer/HikariCP]]]
  :name "puppetdb"
- :plugins [[puppetlabs/lein-ezbake "2.4.1"]]}
+ :plugins [[puppetlabs/lein-ezbake "2.5.3"]]}
  :testutils {:source-paths ^:replace ["test"]
  :resource-paths ^:replace []
  ;; Something else may need adjustment, but

resources/puppetlabs/puppetdb/pql/pql-grammar.ebnf

@@ -52,8 +52,8 @@ extract = <lbracket>, [<whitespace>], [extractfields], [<whitespace>], <rbracket
  after expr is the order of precedence. One should be careful when modifying
  this structure, and ensure that precedence rules continue to be kept. *)
 <expression> = expr-or;
-expr-or = expr-and { <whitespace>, or, <whitespace>, expr-or };
-expr-and = expr-not { <whitespace>, and, <whitespace>, expr-and };
+expr-or = expr-and | expr-and, <whitespace>, or, <whitespace>, expr-or;
+expr-and = expr-not | expr-not, <whitespace>, and, <whitespace>, expr-and;
 expr-not = ( not, [<whitespace>], expr-not ) | expr-rest;
 <expr-rest> = ( <lparens>, [<whitespace>], expression, [<whitespace>], <rparens> ) | condexpression | condexpnull | subquery;
 

src/puppetlabs/puppetdb/command.clj

@@ -295,7 +295,8 @@
  received-time (str (time/to-long received-time))
  duration (str (in-millis (interval start-time (now))))
  command-name (command-names command-kw)
- producer-utc-s (-> producer-ts time/to-timestamp .getTime)
+ producer-utc-s (or (some-> producer-ts time/to-timestamp .getTime)
+ "nil")
  summary (str id "-" received-time "-" producer-utc-s)
  hash-prefix (if hash
  (let [hs (if (bytes? hash)
@@ -528,11 +529,11 @@
  ;; also assuming that all commands don't vary their behavior
  ;; across retries sufficiently for that to matter either.
  ;; This was originally introduced to handle
- ;; program_limit_exceeded errors caused by attemptps to insert
+ ;; program_limit_exceeded errors caused by attempts to insert
  ;; resource titles that were too big to fit in a postgres
  ;; index.
  ;; cf. https://www.postgresql.org/docs/11/errcodes-appendix.html
- (when (str/starts-with? (.getSQLState ex) "54")
+ (when (some-> ex .getSQLState (str/starts-with? "54"))
  (throw (fatality ex)))
  (throw ex)))))
 

src/puppetlabs/puppetdb/pql/transform.clj

@@ -33,13 +33,25 @@
  ;; Single arg? collapse
  ([data] data)
  ;; Multiple args? turn it into an or statement
- ([data & args] (vec (concat ["or" data] args))))
+ ([data & args]
+ (->> (cons data args)
+ (mapcat #(if (and (vector? %) (= (first %) "or"))
+ (rest %)
+ [%]))
+ (cons "or")
+ (vec))))
 
 (defn transform-expr-and
  ;; Single arg? collapse
  ([data] data)
  ;; Multiple args? turn it into an and statement
- ([data & args] (vec (concat ["and" data] args))))
+ ([data & args]
+ (->> (cons data args)
+ (mapcat #(if (and (vector? %) (= (first %) "and"))
+ (rest %)
+ [%]))
+ (cons "and")
+ (vec))))
 
 (defn transform-expr-not
  ;; Single arg? Just collapse the :expr-not and pass back the data,

src/puppetlabs/puppetdb/scf/migrate.clj

@@ -2555,8 +2555,8 @@
  (if-not user
  (do
  (log/info
- (trs "Disconnecting all {0} connections to {1} database before migrating"
- user db-name))
+ (trs "Disconnecting all non-migrator connections to {0} database before migrating"
+ db-name))
  (doseq [user users]
  ;; Because the revoke may not actually produce an error when
  ;; it doesn't work.
@@ -2617,16 +2617,17 @@
  tables))]
  (if-not write-user
  (migrate)
- (call-with-connections-blocked-during-migration
- db-name
- (distinct [read-user write-user])
- (fn []
- ;; So new tables, etc. are owned by the write-user
- (jdbc/do-commands (str "set role " (jdbc/double-quote write-user)))
- (try!
- (migrate)
- (finally
- (jdbc/do-commands (str "set role " (jdbc/double-quote orig-user))))))))))
+ (when-not (empty? (pending-migrations))
+ (call-with-connections-blocked-during-migration
+ db-name
+ (distinct [read-user write-user])
+ (fn []
+ ;; So new tables, etc. are owned by the write-user
+ (jdbc/do-commands (str "set role " (jdbc/double-quote write-user)))
+ (try!
+ (migrate)
+ (finally
+ (jdbc/do-commands (str "set role " (jdbc/double-quote orig-user)))))))))))
 
 (defn initialize-schema
  "Ensures the database is migrated to the latest version, and returns