diff --git a/lib/embedded_ansible.rb b/lib/embedded_ansible.rb
index bd6b7079583..ea5a9613704 100644
--- a/lib/embedded_ansible.rb
+++ b/lib/embedded_ansible.rb
@@ -28,6 +28,7 @@ def self.running?
   end
 
   def self.configured?
+    return false unless File.exist?(SECRET_KEY_FILE)
     key = miq_database.ansible_secret_key
     key.present? && key == File.read(SECRET_KEY_FILE)
   end
@@ -98,6 +99,10 @@ def self.run_setup_script(exclude_tags)
       }
       AwesomeSpawn.run!(SETUP_SCRIPT, :params => params)
     end
+  rescue AwesomeSpawn::CommandResultError => e
+    _log.error("EmbeddedAnsible setup script failed with: #{e.message}")
+    miq_database.ansible_secret_key = nil
+    raise
   end
   private_class_method :run_setup_script
 
diff --git a/spec/lib/embedded_ansible_spec.rb b/spec/lib/embedded_ansible_spec.rb
index 8ed8871fbac..88121025ec9 100644
--- a/spec/lib/embedded_ansible_spec.rb
+++ b/spec/lib/embedded_ansible_spec.rb
@@ -232,6 +232,12 @@
 
           expect(described_class.configured?).to be false
         end
+
+        it "returns false when the file doesn't exist and there is a value in the database" do
+          key_file.unlink
+          miq_database.ansible_secret_key = "password"
+          expect(described_class.configured?).to be false
+        end
       end
 
       describe ".configure_secret_key (private)" do
@@ -256,10 +262,10 @@
       before do
         expect(described_class).to receive(:configured?).and_return(false)
         expect(described_class).to receive(:configure_secret_key)
-        expect(described_class).to receive(:alive?).and_return(true)
       end
 
       it "generates new passwords with no passwords set" do
+        expect(described_class).to receive(:alive?).and_return(true)
         expect(described_class).to receive(:generate_database_authentication).and_return(double(:userid => "awx", :password => "databasepassword"))
         expect(AwesomeSpawn).to receive(:run!) do |script_path, options|
           params                  = options[:params]
@@ -284,6 +290,7 @@
       end
 
       it "uses the existing passwords when they are set in the database" do
+        expect(described_class).to receive(:alive?).and_return(true)
         miq_database.set_ansible_admin_authentication(:password => "adminpassword")
         miq_database.set_ansible_rabbitmq_authentication(:userid => "rabbituser", :password => "rabbitpassword")
         miq_database.set_ansible_database_authentication(:userid => "databaseuser", :password => "databasepassword")
@@ -306,6 +313,15 @@
 
         described_class.start
       end
+
+      it "removes the secret key from the database when setup fails" do
+        miq_database.ansible_secret_key = "supersecretkey"
+        expect(described_class).to receive(:generate_database_authentication).and_return(double(:userid => "awx", :password => "databasepassword"))
+
+        expect(AwesomeSpawn).to receive(:run!).and_raise(AwesomeSpawn::CommandResultError.new("error", 1))
+        expect { described_class.start }.to raise_error(AwesomeSpawn::CommandResultError)
+        expect(miq_database.reload.ansible_secret_key).not_to be_present
+      end
     end
 
     describe ".generate_database_authentication (private)" do