Support for Saved Queries (#862)

Fixes #162 

Adds a new `saved_queries` table, model, and endpoints. These can be used to expose specific queries over public or authed HTTP, for use in public dashboards and websites, where embedding a connection string for direct connection or using 'connection string auth' is unsuitable.

See https://github.com/webhookdb/docs/blob/591f51b4c0ec5ebe3cf1172a71013a6dad21547c/docs/integrating/saved-queries.md for full documentation.

See webhookdb/docs#5 for docs.

See webhookdb/webhookdb-cli#64 for CLI changes.

Also includes some webterm changes:

- Inline input (the one used for prompts) submits automatically when multiple lines are pasted. This was required for pasting SQL to work.
- The input is full width at the beginning (before, it would expand to the widest log line).

db/migrations/039_saved_query.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    create_table(:saved_queries) do
+      primary_key :id
+      timestamptz :created_at, null: false, default: Sequel.function(:now)
+      timestamptz :updated_at
+      foreign_key :organization_id, :organizations, null: false, unique: true, on_delete: :cascade
+      foreign_key :created_by_id, :customers, on_delete: :set_null
+      text :opaque_id, null: false, unique: true
+      text :description, null: false
+      text :sql, null: false
+      boolean :public, null: false, default: false
+    end
+  end
+end

lib/webhookdb.rb

@@ -179,6 +179,17 @@ def self.to_slug(s)
     "9" => "nine",
   }.freeze
 
+  def self.parse_bool(s)
+    # rubocop:disable Style/NumericPredicate
+    return false if s == nil? || s.blank? || s == 0
+    # rubocop:enable Style/NumericPredicate
+    return true if s.is_a?(Integer)
+    sb = s.to_s.downcase
+    return true if ["true", "t", "yes", "y", "on", "1"].include?(sb)
+    return false if ["false", "f", "no", "n", "off", "0"].include?(sb)
+    raise ArgumentError, "unparseable bool: #{s.inspect}"
+  end
+
   # Return the request user and admin stored in TLS. See service.rb for implementation.
   #
   # Note that the second return value (the admin) will be nil if not authed as an admin,

lib/webhookdb/api.rb

@@ -60,13 +60,22 @@ def lookup_org!(identifier=nil, customer: nil, allow_connstr_auth: false)
             return org
           end
 
-          def ensure_admin!(org=nil, customer: nil)
+          # rubocop:disable Naming/PredicateName
+          def has_admin?(org=nil, customer: nil)
+            # rubocop:enable Naming/PredicateName
             customer ||= current_customer
             org ||= lookup_org!
             has_no_admin = org.verified_memberships_dataset.
               where(customer:, membership_role: Webhookdb::Role.admin_role).
               empty?
-            permission_error!("You don't have admin privileges with #{org.name}.") if has_no_admin
+            return !has_no_admin
+          end
+
+          def ensure_admin!(org=nil, customer: nil)
+            org ||= lookup_org!
+            admin = has_admin?(org, customer:)
+            # noinspection RubyNilAnalysis
+            permission_error!("You don't have admin privileges with #{org.name}.") unless admin
           end
         end
 

lib/webhookdb/api/db.rb

@@ -124,25 +124,8 @@ def run_fdw
       end
       post :sql do
         org = lookup_org!(allow_connstr_auth: true)
-        begin
-          r = org.execute_readonly_query(params[:query])
-        rescue Sequel::DatabaseError => e
-          self.logger.error("db_query_database_error", error: e)
-          # We want to handle InsufficientPrivileges and UndefinedTable explicitly
-          # since we can hint the user at what to do.
-          # Otherwise, we should just return the Postgres exception.
-          case e.wrapped_exception
-            when PG::UndefinedTable
-              missing_table = e.wrapped_exception.message.match(/relation (.+) does not/)&.captures&.first
-              msg = "The table #{missing_table} does not exist. Run `webhookdb db tables` to see available tables." if
-                missing_table
-            when PG::InsufficientPrivilege
-              msg = "You do not have permission to perform this query. Queries must be read-only."
-          else
-              msg = e.wrapped_exception.message
-          end
-          merror!(403, msg, code: "invalid_query")
-        end
+        r, msg = execute_readonly_query(org, params[:query])
+        merror!(403, msg, code: "invalid_query") if r.nil?
         status 200
         present({rows: r.rows, headers: r.columns, max_rows_reached: r.max_rows_reached})
       end

lib/webhookdb/api/helpers.rb

@@ -250,4 +250,31 @@ def _log_webhook_request(opaque_id, organization_id, sstatus, request_headers)
       service_integration_opaque_id: opaque_id,
     )
   end
+
+  # Run the given SQL inside the org, and use special error handling if it fails.
+  # @return [Array<Webhookdb::Organization::QueryResult,String,nil>] Tuple of query result, and optional message.
+  #   On query success, return <QueryResult, nil>.
+  #   On DatabaseError, return <nil, message>.
+  #   On other types of errors, raise.
+  def execute_readonly_query(org, sql)
+    result = org.execute_readonly_query(sql)
+    return result, nil
+  rescue Sequel::DatabaseError => e
+    self.logger.error("db_query_database_error", error: e)
+    # We want to handle InsufficientPrivileges and UndefinedTable explicitly
+    # since we can hint the user at what to do.
+    # Otherwise, we should just return the Postgres exception.
+    msg = ""
+    case e.wrapped_exception
+      when PG::UndefinedTable
+        missing_table = e.wrapped_exception.message.match(/relation (.+) does not/)&.captures&.first
+        msg = "The table #{missing_table} does not exist. Run `webhookdb db tables` to see available tables." if
+          missing_table
+      when PG::InsufficientPrivilege
+        msg = "You do not have permission to perform this query. Queries must be read-only."
+      else
+        msg = e.wrapped_exception.message
+    end
+    return [nil, msg]
+  end
 end

lib/webhookdb/api/saved_queries.rb

@@ -0,0 +1,219 @@
+# frozen_string_literal: true
+
+require "webhookdb/api"
+require "webhookdb/saved_query"
+
+class Webhookdb::API::SavedQueries < Webhookdb::API::V1
+  resource :organizations do
+    route_param :org_identifier do
+      resource :saved_queries do
+        helpers do
+          def lookup!
+            org = lookup_org!
+            # We can add other identifiers in the future
+            cq = org.saved_queries_dataset[opaque_id: params[:query_identifier]]
+            merror!(403, "There is no saved query with that identifier.") if cq.nil?
+            return cq
+          end
+
+          def guard_editable!(customer, cq)
+            return if customer === cq.created_by
+            return if has_admin?(cq.organization, customer:)
+            permission_error!("You must be the query's creator or an org admin.")
+          end
+
+          def execute_readonly_query_with_suggestion(org, sql)
+            r, message = execute_readonly_query(org, sql)
+            return r, nil unless r.nil?
+            msg = "Something went wrong running your query. Perhaps a table it depends on was deleted. " \
+                  "Check out #{Webhookdb::SavedQuery::DOCS_URL} for troubleshooting tips. " \
+                  "Here's what went wrong: #{message}"
+            return r, msg
+          end
+        end
+
+        desc "Returns a list of all custom queries associated with the org."
+        get do
+          queries = lookup_org!.saved_queries
+          message = ""
+          if queries.empty?
+            message = "This organization doesn't have any saved queries yet.\n" \
+                      "Use `webhookdb saved-query create` to set one up."
+          end
+          present_collection queries, with: SavedQueryEntity, message:
+        end
+
+        desc "Creates a custom query."
+        params do
+          optional :description, type: String, prompt: "What is the query used for? "
+          optional :sql, type: String, prompt: "Enter the SQL you would like to run: "
+          optional :public, type: Boolean
+        end
+        post :create do
+          cust = current_customer
+          org = lookup_org!
+          _, errmsg = execute_readonly_query_with_suggestion(org, params[:sql])
+          if errmsg
+            Webhookdb::API::Helpers.prompt_for_required_param!(
+              request,
+              :sql,
+              "Enter a new query:",
+              output: "That query was invalid. #{errmsg}\n" \
+                      "You can iterate on your query by connecting to your database from any SQL editor.\n" \
+                      "Use `webhookdb db connection` to get your query string.",
+            )
+          end
+          cq = Webhookdb::SavedQuery.create(
+            description: params[:description],
+            sql: params[:sql],
+            organization: org,
+            created_by: cust,
+            public: params[:public] || false,
+          )
+          message = "You have created a new saved query with an id of '#{cq.opaque_id}'. " \
+                    "You can run it through the CLI, or through the API with or without authentication. " \
+                    "See #{Webhookdb::SavedQuery::DOCS_URL} for more information."
+          status 200
+          present cq, with: SavedQueryEntity, message:
+        end
+
+        route_param :query_identifier, type: String do
+          desc "Returns the query with the given identifier."
+          get do
+            cq = lookup!
+            status 200
+            message = "See #{Webhookdb::SavedQuery::DOCS_URL} to see how to run or modify your query."
+            present cq, with: SavedQueryEntity, message:
+          end
+
+          desc "Runs the query with the given identifier."
+          get :run do
+            _customer = current_customer
+            org = lookup_org!
+            cq = lookup!
+            r, msg = execute_readonly_query_with_suggestion(org, cq.sql)
+            merror!(400, msg) if r.nil?
+            status 200
+            present({rows: r.rows, headers: r.columns, max_rows_reached: r.max_rows_reached})
+          end
+
+          desc "Updates the field on a custom query."
+          params do
+            optional :field, type: String, prompt: "What field would you like to update (one of: " \
+                                                   "#{Webhookdb::SavedQuery::CLI_EDITABLE_FIELDS.join(', ')}): "
+            optional :value, type: String, prompt: "What is the new value? "
+          end
+          post :update do
+            customer = current_customer
+            cq = lookup!
+            guard_editable!(customer, cq)
+            # Instead of specifying which values are valid for the optional `field` param in the param declaration,
+            # we do the validation here so that we can provide a more helpful error message
+            unless Webhookdb::SavedQuery::CLI_EDITABLE_FIELDS.include?(params[:field])
+              merror!(400, "That field is not editable.")
+            end
+            value = params[:value]
+            case params[:field]
+              when "public"
+                begin
+                  value = Webhookdb.parse_bool(value)
+                rescue ArgumentError => e
+                  Webhookdb::API::Helpers.prompt_for_required_param!(
+                    request,
+                    :value,
+                    e.message + "\nAny boolean-like string (true, false, yes, no, etc) will work:",
+                  )
+                end
+                cq.public = value
+              when "sql"
+                r, msg = execute_readonly_query_with_suggestion(cq.organization, value)
+                if r.nil?
+                  Webhookdb::API::Helpers.prompt_for_required_param!(
+                    request,
+                    :value,
+                    "Enter your query:",
+                    output: msg,
+                  )
+                end
+                cq.sql = value
+              else
+                cq.send(:"#{params[:field]}=", value)
+            end
+            cq.save_changes
+            status 200
+            # Do not render the value here, it can be very long.
+            message = "You have updated '#{params[:field]}' on saved query '#{cq.opaque_id}'."
+            present cq, with: SavedQueryEntity, message:
+          end
+
+          params do
+            optional :field, type: String, values: Webhookdb::SavedQuery::INFO_FIELDS.keys + [""]
+          end
+          post :info do
+            cq = lookup!
+            data = Webhookdb::SavedQuery::INFO_FIELDS.
+              to_h { |k, v| [k.to_sym, cq.send(v)] }
+
+            field_name = params[:field]
+            blocks = Webhookdb::Formatting.blocks
+            if field_name.present?
+              blocks.line(data.fetch(field_name.to_sym))
+            else
+              rows = data.map do |k, v|
+                [k.to_s.humanize, v.to_s]
+              end
+              blocks.table(["Field", "Value"], rows)
+            end
+            r = {blocks: blocks.as_json}
+            status 200
+            present r
+          end
+
+          post :delete do
+            customer = current_customer
+            cq = lookup!
+            guard_editable!(customer, cq)
+            cq.destroy
+            status 200
+            present cq, with: SavedQueryEntity,
+                        message: "You have successfully deleted the saved query '#{cq.description}'."
+          end
+        end
+      end
+    end
+  end
+
+  resource :saved_queries do
+    route_param :query_identifier, type: String do
+      get :run do
+        # This endpoint can be used publicly, so should expose as little information as possible.
+        # Do not expose permissions or query details.
+        cq = Webhookdb::SavedQuery[opaque_id: params[:query_identifier]]
+        forbidden! if cq.nil?
+        if cq.private?
+          authed = Webhookdb::API::ConnstrAuth.find_authed([cq.organization], request)
+          if !authed && (cust = current_customer?)
+            authed = !cust.verified_memberships_dataset.where(organization: cq.organization).empty?
+            end
+          forbidden! unless authed
+        end
+        r, _ = execute_readonly_query(cq.organization, cq.sql)
+        merror!(400, "Something went wrong running the query.") if r.nil?
+        status 200
+        present({rows: r.rows, headers: r.columns, max_rows_reached: r.max_rows_reached})
+      end
+    end
+  end
+
+  class SavedQueryEntity < Webhookdb::API::BaseEntity
+    expose :opaque_id, as: :id
+    expose :description
+    expose :sql
+    expose :public
+    expose :run_url
+
+    def self.display_headers
+      return [[:id, "Id"], [:description, "Description"], [:public, "Public"], [:run_url, "Run URL"]]
+    end
+  end
+end

lib/webhookdb/apps.rb

@@ -17,6 +17,7 @@
 require "webhookdb/api/me"
 require "webhookdb/api/organizations"
 require "webhookdb/api/replay"
+require "webhookdb/api/saved_queries"
 require "webhookdb/api/service_integrations"
 require "webhookdb/api/services"
 require "webhookdb/api/stripe"
@@ -68,6 +69,7 @@ class API < Webhookdb::Service
     mount Webhookdb::API::Me
     mount Webhookdb::API::Organizations
     mount Webhookdb::API::Replay
+    mount Webhookdb::API::SavedQueries
     mount Webhookdb::API::ServiceIntegrations
     mount Webhookdb::API::Services
     mount Webhookdb::API::Stripe

lib/webhookdb/fixtures/saved_queries.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "faker"
+
+require "webhookdb"
+require "webhookdb/fixtures"
+
+module Webhookdb::Fixtures::SavedQueries
+  extend Webhookdb::Fixtures
+
+  fixtured_class Webhookdb::SavedQuery
+
+  base :saved_query do
+    self.description ||= Faker::Lorem.sentence
+    self.sql ||= "SELECT * FROM mytable"
+  end
+
+  before_saving do |instance|
+    instance.organization ||= Webhookdb::Fixtures.organization.create
+    instance
+  end
+
+  decorator :created_by do |c={}|
+    c = Webhookdb::Fixtures.customer.create(c) unless c.is_a?(Webhookdb::Customer)
+    self.created_by = c
+  end
+end

lib/webhookdb/organization.rb

@@ -34,6 +34,7 @@ class SchemaMigrationError < StandardError; end
               adder: ->(om) { om.update(organization_id: id, verified: false) },
               order: :id
   one_to_many :service_integrations, class: "Webhookdb::ServiceIntegration", order: :id
+  one_to_many :saved_queries, class: "Webhookdb::SavedQuery", order: :id
   one_to_many :webhook_subscriptions, class: "Webhookdb::WebhookSubscription", order: :id
   many_to_many :feature_roles, class: "Webhookdb::Role", join_table: :feature_roles_organizations, right_key: :role_id
   one_to_many :all_webhook_subscriptions,