Add DB encryption for email + OpenAI and Anthropic API keys (#274)

.github/workflows/rubyonrails.yml

@@ -37,7 +37,7 @@ jobs:
           bundler-cache: true
 
       - name: Set up database schema
-        run: bin/rails db:schema:load
+        run: bin/rails db:prepare
 
       - name: Build CSS
         run: bin/rails tailwindcss:build
@@ -85,7 +85,7 @@ jobs:
           bundler-cache: true
 
       - name: Set up database schema
-        run: bin/rails db:schema:load
+        run: bin/rails db:prepare
 
       - name: Build CSS
         run: bin/rails tailwindcss:build

.gitignore

@@ -33,6 +33,7 @@
 
 # Ignore master key for decrypting credentials and more.
 /config/master.key
+/config/credentials.yml.enc
 
 /app/assets/builds/*
 !/app/assets/builds/.keep

README.md

@@ -68,30 +68,33 @@ We welcome contributors! After you get your developoment environment setup, revi
 
 The easiest way to get up and running is to use the provided docker compose workflow. The only things you need installed on your computer are Docker and Git.
 
-1. Make sure you have [Docker Desktop](https://docs.docker.com/desktop/) installed and running.
+1. Make sure you have [Docker Desktop](https://docs.docker.com/desktop/) installed and running
 2. Clone your fork `git clone [repository url]`
-3. `cd` into your clone.
-4. Run `docker compose up` to start the app.
-5. Open [http://localhost:3000](http://localhost:3000) and register as a new user.
-6. Run tests: `docker compose run base rails test` The app has comprehensive test coverage.
+3. `cd` into your clone
+4. Run `docker compose up --build` to start the app
+5. Open [http://localhost:3000](http://localhost:3000) and register as a new user
+6. Run tests: `docker compose run base rails test` The app has comprehensive test coverage but note that system tests currently do not work in docker.
 7. Open the rails console: `docker compose run base rails console`
 8. Run a psql console: `docker compose run base psql`
 
-Alternatively, you can set up your development environment locally:
+Every time you pull new changes down, kill docker (if it's running) and re-run:
+`docker compose up --build` This will ensure your local app picks up changes to Gemfile, migrations, and docker config.
+
+### Alternatively, you can set up your development environment locally:
 
 HostedGPT requires these services to be running:
 
 - Postgres ([installation instructions](https://www.postgresql.org/download/))
 - Redis ([installation instructions](https://redis.io/download))
-- asdf-vm ([installation instructions](https://asdf-vm.com/guide/getting-started.html#_2-download-asdf))
+- rbenv or asdf-vm ([installation instructions](https://asdf-vm.com/guide/getting-started.html#_2-download-asdf))
 
 1. `cd` into your local repository clone
-2. `asdf install` to install the correct ruby version
-3. `bundle install` to install ruby gems
-4. `bin/rails db:setup` < Note: This will load the sample fixture data into your database
-5. `bin/dev` < Starts up all the services
-6. Open [http://localhost:3000](http://localhost:3000) and register as a new user
-7. `bin/rails test` and `bin/rails test:system` to run the comprehensive tests
+2. `rbenv install` or `asdf install` to install the correct ruby version
+3. `bin/dev` starts up all the services, installs gems, and inits database (don't run **db:setup** as it will not configure encryption properly)
+4. Open [http://localhost:3000](http://localhost:3000) and register as a new user
+5. `bin/rails test` and `bin/rails test:system` to run the comprehensive tests
+
+Every time you pull new changes down, kill `bin/dev` and then re-run it. This will ensure your local app picks up changes to Gemfile and migrations.
 
 # Changelog
 
@@ -119,3 +122,4 @@ v0.5 - Released on 2/14/2024
 * Conversations are automatically titled
 * Sidebar can be closed
 * AI responses stream in
+>>>>>>> upstream-main

app/models/person.rb

@@ -1,4 +1,6 @@
 class Person < ApplicationRecord
+  encrypts :email, deterministic: true
+
   delegated_type :personable, types: %w[User Tombstone]
   accepts_nested_attributes_for :personable
 

app/models/user.rb

@@ -1,5 +1,6 @@
 class User < ApplicationRecord
   include Personable, Registerable
+  encrypts :openai_key, :anthropic_key
 
   has_secure_password
   has_person_name

bin/dev

@@ -9,5 +9,8 @@ else
     echo "Installing foreman..."
     gem install foreman
   fi
+
+  bundle install
+  bin/rails db:prepare
   exec foreman start -f Procfile.dev "$@"
 fi

bin/docker-entrypoint

@@ -1,10 +1,21 @@
 #!/bin/sh -e
 
 # If running the rails server then create or migrate existing database
-if [ "${1}" == "./bin/rails" ] && [ "${2}" == "server" ]; then
+if echo "${1}" | grep -q "rails$" && [ "${2}" == "server" ]; then
+  rm -f ./tmp/pids/server.pid 2>/dev/null
+  echo "Running bundle"
+  bundle
+  echo "Running db:prepare"
+  ./bin/rails db:prepare
+elif echo "${1}" | grep -q "rails$" && [ "${2}" == "test" ]; then
+  export RAILS_ENV=test
+  echo "Running bundle"
+  bundle
   echo "Running db:prepare"
   ./bin/rails db:prepare
 else
+  echo "Running bundle"
+  bundle
   echo "Skipping db:prepare"
 fi
 

config/application.rb

@@ -11,6 +11,9 @@ class Application < Rails::Application
     # Initialize configuration defaults for originally generated Rails version.
     config.load_defaults 7.1
 
+    # To remove in 2025. This allows migration db/migrate/20240415134849_encrypt_keys.rb to encrypt existing plaintext keys
+    config.active_record.encryption.support_unencrypted_data = true
+
     # Please, add to the `ignore` list any other `lib` subdirectories that do
     # not contain `.rb` files, or that should not be reloaded or eager loaded.
     # Common ones are `templates`, `generators`, or `middleware`, for example.

config/environments/test.rb

@@ -23,6 +23,8 @@
     "Cache-Control" => "public, max-age=#{1.hour.to_i}"
   }
 
+  config.active_record.encryption.encrypt_fixtures = true
+
   # Show full error reports and disable caching.
   config.consider_all_requests_local = true
   config.action_controller.perform_caching = false

config/initializers/active_record_encryption.rb

@@ -0,0 +1,10 @@
+# Commonly configured through config/credentials.yml.enc.
+# For deployment to Render, we support configuration through ENV variables
+if ENV['CONFIGURE_ACTIVE_RECORD_ENCRYPTION_FROM_ENV'] == 'true'
+  Rails.application.configure do
+    Rails.logger.info "Configuring active record encryption from environment"
+    config.active_record.encryption.primary_key = ENV['ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY']
+    config.active_record.encryption.deterministic_key = ENV['ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY']
+    config.active_record.encryption.key_derivation_salt = ENV['ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT']
+  end
+end

db/migrate/20240425173452_encrypt_keys.rb

@@ -0,0 +1,23 @@
+class EncryptKeys < ActiveRecord::Migration[7.1]
+  def up
+    puts ""
+    puts "### ERROR? #################################################"
+    puts "### YOU SHOULD NOT RUN db:migrate INSTEAD RUN db:prepare ###"
+    puts "############################################################"
+    puts ""
+
+    User.find_each do |user|
+      Rails.logger.info "Encrypt keys for #{user.id}. Has openai_key: #{user.openai_key.present?}; has anthropic_key: #{user.anthropic_key.present?}"
+      user.encrypt
+      if !user.save
+        Rails.logger.warn "Could not update user #{user.id}: #{user.errors.full_messages.join(',')}"
+      else
+        Rails.logger.info "Successfully updated user #{user.id}"
+      end
+    end
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration.new "Won't decrypt data"
+  end
+end

db/migrate/20240425173453_encrypt_person_emails.rb

@@ -0,0 +1,17 @@
+class EncryptPersonEmails < ActiveRecord::Migration[7.1]
+  def up
+    Person.find_each do |person|
+      Rails.logger.info "Encrypt email for #{person.id}"
+      person.encrypt
+      if !person.save(validate: false)
+        Rails.logger.warn "Could not update person #{person.id}: #{person.errors.full_messages.join(',')}"
+      else
+        Rails.logger.info "Successfully updated user #{person.id}"
+      end
+    end
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration.new "Won't decrypt data"
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.1].define(version: 2024_04_18_162137) do
+ActiveRecord::Schema[7.1].define(version: 2024_04_25_173453) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 

lib/tasks/prepare.rake

@@ -0,0 +1,62 @@
+namespace :db do
+  desc "Setup database encryption and update credentials"
+  task setup_encryption: :environment do
+    ensure_master_key
+
+    old_config = Rails.application.credentials.config
+    config = old_config.deep_dup
+
+    if config[:secret_key_base].nil? && ENV['SECRET_KEY_BASE'].nil?
+      config = add_secret_key_base(config)
+    end
+
+    if config[:active_record_encryption].nil? && ENV['CONFIGURE_ACTIVE_RECORD_ENCRYPTION_FROM_ENV'] != 'true'
+      config = add_active_record_encryption(config)
+    end
+
+    if config != old_config
+      Rails.application.credentials.write(config.to_yaml)
+      ActiveRecord::Encryption.config.primary_key = config[:active_record_encryption][:primary_key]
+      ActiveRecord::Encryption.config.deterministic_key = config[:active_record_encryption][:deterministic_key]
+      ActiveRecord::Encryption.config.key_derivation_salt = config[:active_record_encryption][:key_derivation_salt]
+    end
+  end
+end
+
+Rake::Task["db:prepare"].enhance [:setup_encryption]
+
+def add_secret_key_base(config)
+  config[:secret_key_base] = SecureRandom.hex(64)
+  config
+end
+
+def add_active_record_encryption(config)
+  config[:active_record_encryption] = {
+    primary_key: SecureRandom.alphanumeric(32),
+    deterministic_key: SecureRandom.alphanumeric(32),
+    key_derivation_salt: SecureRandom.alphanumeric(32),
+  }
+  config
+end
+
+def encryption_init
+  original_stdout = $stdout
+  $stdout = StringIO.new
+  Rake::Task["db:encryption:init"].invoke
+  output = $stdout.string
+  $stdout = original_stdout
+
+  {
+    primary_key: output.match(/primary_key: (\S+)/)[1],
+    deterministic_key: output.match(/deterministic_key: (\S+)/)[1],
+    key_derivation_salt: output.match(/key_derivation_salt: (\S+)/)[1],
+  }
+end
+
+def ensure_master_key
+  master_key_path = Rails.root.join('config', 'master.key')
+  unless File.exist?(master_key_path)
+    key = SecureRandom.hex(16)
+    File.write(master_key_path, key)
+  end
+end

render.yaml

@@ -9,6 +9,14 @@ envVarGroups:
   envVars:
     - key: SECRET_KEY_BASE
       generateValue: true
+    - key: CONFIGURE_ACTIVE_RECORD_ENCRYPTION_FROM_ENV
+      value: true
+    - key: ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY
+      generateValue: true
+    - key: ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY
+      generateValue: true
+    - key: ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT
+      generateValue: true
 
 services:
   - type: web

test/models/person_test.rb

@@ -5,6 +5,27 @@ class PersonTest < ActiveSupport::TestCase
     assert_instance_of User, people(:keith_registered).user
   end
 
+  test "encrypts email" do
+    person = people(:rob_registered)
+    old_email = person.email
+    old_cipher_text = person.ciphertext_for(:email)
+    person.update!(email: "new@address.net")
+    assert person.reload
+    refute_equal old_cipher_text, person.ciphertext_for(:email)
+    assert_equal "new@address.net", person.email
+  end
+
+  # Appears length limit for email addresses is 256
+  test "encrypts long emails" do
+    user = User.new password: "password", first_name: "John", last_name: "Doe"
+    long_email_address = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@xxx.net"
+    assert_equal 256, long_email_address.length
+    person = Person.new email: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@xxx.net", personable: user
+    person.save!
+    person.reload
+    assert_equal long_email_address, person.email
+  end
+
   test "requires email addresses to be unique" do
     person1 = users(:keith).person
     person2 = Person.new(email: person1.email)

test/models/user_test.rb

@@ -11,6 +11,26 @@ class UserTest < ActiveSupport::TestCase
     refute person.valid?
   end
 
+  test "encrypts openai_key" do
+    user = users(:keith)
+    old_openai_key = user.openai_key
+    old_cipher_text = user.ciphertext_for(:openai_key)
+    user.update!(openai_key: "new one")
+    assert user.reload
+    refute_equal old_cipher_text, user.ciphertext_for(:openai_key)
+    assert_equal "new one", user.openai_key
+  end
+
+  test "encrypts anthropic_key" do
+    user = users(:keith)
+    old_anthropic_key = user.anthropic_key
+    old_cipher_text = user.ciphertext_for(:anthropic_key)
+    user.update!(anthropic_key: "new one")
+    assert user.reload
+    refute_equal old_cipher_text, user.ciphertext_for(:anthropic_key)
+    assert_equal "new one", user.anthropic_key
+  end
+
   test "should validate a user with minimum information" do
     user = User.new(password: "password", password_confirmation: "password", first_name: "John", last_name: "Doe")
     person = Person.new(email: "example@gmail.com", personable: user)