(PE-36193) PDB benchmark can use ssl

documentation/load_testing_tool.markdown

@@ -65,14 +65,35 @@ which may significantly skew performance on the primary server. If you would lik
 the benchmark tool on an agent this can be achieved following the instructions
 below.
 
+#### Running over https
+
+For authentication, you will use the agent's Puppet certificates from /etc/puppetlabs/puppet/ssl.
+
+* On the primary server, modify `/etc/puppetlabs/puppetdb/certificate-allowlist` to include the agent's certificate name (the host fqdn).
+* On the agent, modify the config.ini you will use with the bechmark tool to instead provide ssl host/port and certificate information:
+
+      [jetty]
+      ssl-host=<host name here>
+      ssl-port=<ssl port here (defaults to 8081)>
+      ssl-cert=<path to the agent's /etc/puppetlabs/puppet/ssl/certs pem file>
+      ssl-key=<path to the agent's /etc/puppetlabs/puppet/ssl/private_keys pem file>
+      ssl-ca-cert=/etc/puppetlabs/puppet/ssl/certs/ca.pem
+
+* Install java on the agent
+
+After these steps have been completed you should be able to run the benchmark
+tool on the agent using the `java -cp ...` command described above.
+
+#### Running over http *(insecure)*
+
+This is not recommended, as the configuration change will allow http
+connections from *any* source.
+
 * On the primary server, modify `/etc/puppetlabs/puppetdb/conf.d/jetty.ini`.
-In the `[jetty]` section, set either:
-    * `host=0.0.0.0 # http access from all agents`
-    * `host=<agent ip address> # access from specific agent`
+In the `[jetty]` section set:
+    * `host=0.0.0.0 # open http access`
 
 * Install java on the agent
-* On the agent, in the `config.ini` file set the port to the puppetdb port for
-http traffic (defaults to 8080)
 
 After these steps have been completed you should be able to run the benchmark
 tool on the agent using the `java -cp ...` command described above.

src/puppetlabs/puppetdb/cli/benchmark.clj

@@ -285,7 +285,7 @@
   command-send-ch and sends commands to the puppetdb at base-url. Writes
   ::submitted to rate-monitor-ch for every command sent, or ::error if there was
   a problem. Close command-send-ch to stop the background process."
-  [base-url command-send-ch rate-monitor-ch num-threads]
+  [base-url command-send-ch rate-monitor-ch num-threads ssl-opts]
   (let [fanout-commands-ch (chan)]
     ;; fanout: given a single host state, emit 3 messages, one for each command.
     ;; This gives better parallelism for message submission.
@@ -310,7 +310,7 @@
                               :report client/submit-report
                               :factset client/submit-facts)]
               (try
-                (submit-fn base-url host version payload)
+                (submit-fn base-url host version payload ssl-opts)
                 ::submitted
                 (catch Exception e
                   (println-err (trs "Exception while submitting command: {0}" e))
@@ -510,9 +510,12 @@
         _ (logutils/configure-logging! (get-in config [:global :logging-config]))
         {:keys [catalogs reports facts]} (load-data-from-options options)
         _ (warn-missing-data catalogs reports facts)
-        {pdb-host :host pdb-port :port
-         :or {pdb-host "127.0.0.1" pdb-port 8080}} (:jetty config)
-        base-url (utils/pdb-cmd-base-url pdb-host pdb-port :v1)
+        {:keys [host port ssl-host ssl-port]} (:jetty config)
+        pdb-host (or ssl-host host "127.0.0.1")
+        pdb-port (or ssl-port port "8081")
+        protocol (if ssl-host "https" "http")
+        ssl-opts (select-keys (:jetty config) [:ssl-cert :ssl-key :ssl-ca-cert])
+        base-url (utils/pdb-cmd-base-url pdb-host pdb-port :v1 protocol)
         run-interval (-> (get options :runinterval 30) time/minutes)
         simulation-threads 4
         commands-per-puppet-run (+ (if catalogs 1 0)
@@ -549,7 +552,8 @@
         command-sender-finished-ch (start-command-sender base-url
                                                          command-send-ch
                                                          rate-monitor-ch
-                                                         threads)
+                                                         threads
+                                                         ssl-opts)
         _ (start-simulation-loop numhosts run-interval nummsgs end-commands-in rand-perc
                                  simulation-threads simulation-write-ch simulation-read-ch)
         join-fn (fn join-benchmark

src/puppetlabs/puppetdb/client.clj

@@ -1,6 +1,6 @@
 (ns puppetlabs.puppetdb.client
   (:require [clojure.tools.logging :as log]
-            [clj-http.client :as http-client]
+            [puppetlabs.http.client.sync :as http-client]
             [puppetlabs.puppetdb.command.constants :refer [command-names]]
             [puppetlabs.puppetdb.cheshire :as json]
             [puppetlabs.puppetdb.schema :refer [defn-validated]]
@@ -30,13 +30,21 @@
     command :- s/Str
     version :- s/Int
     payload]
-   (submit-command-via-http! base-url certname command version payload nil))
+   (submit-command-via-http! base-url certname command version payload nil {}))
   ([base-url
     certname :- s/Str
     command :- s/Str
     version :- s/Int
-    payload :- {s/Any s/Any}
+    payload
     timeout]
+   (submit-command-via-http! base-url certname command version payload timeout {}))
+  ([base-url
+    certname :- s/Str
+    command :- s/Str
+    version :- s/Int
+    payload :- {s/Any s/Any}
+    timeout
+    ssl-opts :- {s/Keyword s/Str}]
    (let [body (json/generate-string payload)
          url-params (utils/cmd-url-params {:command command
                                            :version version
@@ -45,26 +53,33 @@
                                                                    :producer_timestamp
                                                                    str)
                                            :timeout timeout})
-         url (str (utils/base-url->str base-url) url-params)]
-     (http-client/post url {:body body
-                            :throw-exceptions false
-                            :content-type :json
-                            :character-encoding "UTF-8"
-                            :accept :json}))))
+         url (str (utils/base-url->str base-url) url-params)
+         post-opts (merge {:body body
+                           :as :text
+                           :headers {"Content-Type" "application/json"}}
+                    ;       :throw-exceptions false
+                    ;       :content-type :json
+                    ;       :character-encoding "UTF-8"
+                    ;       :accept :json}
+                          (select-keys ssl-opts [:ssl-cert :ssl-key :ssl-ca-cert]))]
+     (http-client/post url post-opts))))
 
 (defn-validated submit-catalog
   "Send the given wire-format `catalog` (associated with `host`) to a
   command-processing endpoint located at `puppetdb-host`:`puppetdb-port`."
   [base-url :- utils/base-url-schema
    certname :- s/Str
    command-version :- s/Int
-   catalog-payload]
+   catalog-payload
+   ssl-opts]
   (let [result (submit-command-via-http!
                  base-url
                  certname
                  (command-names :replace-catalog)
                  command-version
-                 catalog-payload)]
+                 catalog-payload
+                 nil
+                 ssl-opts)]
     (when-not (= HttpURLConnection/HTTP_OK (:status result))
       (log/error result))))
 
@@ -74,28 +89,39 @@
   [base-url :- utils/base-url-schema
    certname :- s/Str
    command-version :- s/Int
-   report-payload]
+   report-payload
+   ssl-opts]
   (let [result (submit-command-via-http!
                  base-url
                  certname
                  (command-names :store-report)
                  command-version
-                 report-payload)]
+                 report-payload
+                 nil
+                 ssl-opts)]
     (when-not (= HttpURLConnection/HTTP_OK (:status result))
       (log/error result))))
 
 (defn-validated submit-facts
   "Send the given wire-format `facts` (associated with `host`) to a
    command-processing endpoint located at `puppetdb-host`:`puppetdb-port`."
-  [base-url :- utils/base-url-schema
-   certname :- s/Str
-   facts-version :- s/Int
-   fact-payload]
-  (let [result  (submit-command-via-http!
-                  base-url
-                  certname
-                  (command-names :replace-facts)
-                  facts-version
-                  fact-payload)]
-    (when-not (= HttpURLConnection/HTTP_OK (:status result))
-      (log/error result))))
+  ([base-url :- utils/base-url-schema
+    certname :- s/Str
+    facts-version :- s/Int
+    fact-payload]
+   (submit-facts base-url certname facts-version fact-payload {}))
+  ([base-url :- utils/base-url-schema
+    certname :- s/Str
+    facts-version :- s/Int
+    fact-payload
+    ssl-opts]
+   (let [result  (submit-command-via-http!
+                   base-url
+                   certname
+                   (command-names :replace-facts)
+                   facts-version
+                   fact-payload
+                   nil
+                   ssl-opts)]
+     (when-not (= HttpURLConnection/HTTP_OK (:status result))
+       (log/error result)))))

src/puppetlabs/puppetdb/utils.clj

@@ -245,38 +245,14 @@
             (flush-and-exit 0))
           (throw ex))))))
 
-(defn pdb-query-base-url
-  [host port & [version]]
-  {:protocol "http"
-   :host host
-   :port port
-   :prefix "/pdb/query"
-   :version (or version :v4)})
-
-(defn pdb-admin-base-url
-  [host port & [version]]
-  {:protocol "http"
-   :host host
-   :port port
-   :prefix "/pdb/admin"
-   :version (or version :v1)})
-
 (defn pdb-cmd-base-url
-  [host port & [version]]
-  {:protocol "http"
+  [host port & [version protocol]]
+  {:protocol (or protocol "http")
    :host host
    :port port
    :prefix "/pdb/cmd"
    :version (or version :v1)})
 
-(defn pdb-meta-base-url
-  [host port & [version]]
-  {:protocol "http"
-   :host host
-   :port port
-   :prefix "/pdb/meta"
-   :version (or version :v1)})
-
 (defn metrics-base-url
   [host port & [version]]
   {:protocol "http"

test/puppetlabs/puppetdb/cli/benchmark_test.clj

@@ -21,13 +21,14 @@
    [java.nio.file Files]))
 
 (defn mock-submit-record-fn [submitted-records entity]
-  (fn [base-url _certname version payload-string]
+  (fn [base-url _certname version payload-string ssl-opts]
     (swap! submitted-records conj
            {:entity entity
             :base-url base-url
             :version version
             :payload-string payload-string
-            :payload (keywordize-keys payload-string)})))
+            :payload (keywordize-keys payload-string)
+            :ssl-opts ssl-opts})))
 
 (defn call-with-benchmark-status
   [config cli-args f]
@@ -224,8 +225,8 @@
        (add-watch submitted watch-key watcher)
        (when-not (>= (count @submitted) enough-records) ; avoid add-watch race
          (deref finished tu/default-timeout-ms nil))
-       ;; Allow a ~30% margin of error to account for jitter in the simulation
+       ;; Allow a ~33% margin of error to account for jitter in the simulation
        ;; timer.
        (let [elapsed (/ (- (System/currentTimeMillis) start) 1000.0)]
-         (is (<= 2.1 elapsed 3.9)))
+         (is (<= 2 elapsed 4)))
        (stop)))))