diff --git a/validator/keymanager/v2/direct/BUILD.bazel b/validator/keymanager/v2/direct/BUILD.bazel
index 3efb7c7fdf61..8430ff07f81d 100644
--- a/validator/keymanager/v2/direct/BUILD.bazel
+++ b/validator/keymanager/v2/direct/BUILD.bazel
@@ -27,7 +27,6 @@ go_library(
         "//shared/interop:go_default_library",
         "//shared/params:go_default_library",
         "//shared/petnames:go_default_library",
-        "//shared/promptutil:go_default_library",
         "//validator/accounts/v2/iface:go_default_library",
         "//validator/keymanager/v2:go_default_library",
         "@com_github_fsnotify_fsnotify//:go_default_library",
diff --git a/validator/keymanager/v2/direct/direct.go b/validator/keymanager/v2/direct/direct.go
index 39dc6c116b38..a6a1b8aad9fe 100644
--- a/validator/keymanager/v2/direct/direct.go
+++ b/validator/keymanager/v2/direct/direct.go
@@ -3,7 +3,6 @@ package direct
 import (
 	"bytes"
 	"context"
-	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -23,7 +22,6 @@ import (
 	"github.com/prysmaticlabs/prysm/shared/interop"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/petnames"
-	"github.com/prysmaticlabs/prysm/shared/promptutil"
 	"github.com/prysmaticlabs/prysm/validator/accounts/v2/iface"
 	v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2"
 	"github.com/sirupsen/logrus"
@@ -385,13 +383,7 @@ func (dr *Keymanager) initializeAccountKeystore(ctx context.Context) error {
 	decryptor := keystorev4.New()
 	enc, err := decryptor.Decrypt(keystoreFile.Crypto, password)
 	if err != nil && strings.Contains(err.Error(), "invalid checksum") {
-		// If the password fails for an individual account, we ask the user to input
-		// that individual account's password until it succeeds.
-		enc, password, err = askUntilPasswordConfirms(decryptor, keystoreFile)
-		if err != nil {
-			return errors.Wrap(err, "could not confirm password via prompt")
-		}
-		dr.wallet.SetPassword(password) // Write the correct password to the wallet.
+		return errors.Wrap(err, "wrong password for wallet entered")
 	} else if err != nil {
 		return errors.Wrap(err, "could not decrypt keystore")
 	}
@@ -474,37 +466,3 @@ func (dr *Keymanager) createAccountsKeystore(
 		Name:    encryptor.Name(),
 	}, nil
 }
-
-func askUntilPasswordConfirms(
-	decryptor *keystorev4.Encryptor, keystore *v2keymanager.Keystore,
-) ([]byte, string, error) {
-	au := aurora.NewAurora(true)
-	// Loop asking for the password until the user enters it correctly.
-	var secretKey []byte
-	var password string
-	var err error
-	publicKey, err := hex.DecodeString(keystore.Pubkey)
-	if err != nil {
-		return nil, "", errors.Wrap(err, "could not decode public key")
-	}
-	formattedPublicKey := fmt.Sprintf("%#x", bytesutil.Trunc(publicKey))
-	for {
-		password, err = promptutil.PasswordPrompt(
-			fmt.Sprintf("\nPlease try again, incorrect password for account %s", au.BrightGreen(formattedPublicKey)),
-			promptutil.NotEmpty,
-		)
-		if err != nil {
-			return nil, "", fmt.Errorf("could not read account password: %v", err)
-		}
-		secretKey, err = decryptor.Decrypt(keystore.Crypto, password)
-		if err != nil && strings.Contains(err.Error(), "invalid checksum") {
-			fmt.Print(au.Red("Incorrect password entered, please try again"))
-			continue
-		}
-		if err != nil {
-			return nil, "", err
-		}
-		break
-	}
-	return secretKey, password, nil
-}
diff --git a/validator/rpc/auth.go b/validator/rpc/auth.go
index 8a6695e40f02..a775b25c9a8d 100644
--- a/validator/rpc/auth.go
+++ b/validator/rpc/auth.go
@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/dgrijalva/jwt-go"
@@ -67,7 +68,10 @@ func (s *Server) Login(ctx context.Context, req *pb.AuthRequest) (*pb.AuthRespon
 		WalletDir:      defaultWalletPath,
 		WalletPassword: req.Password,
 	}); err != nil {
-		return nil, status.Error(codes.Internal, "Could not initialize wallet")
+		if strings.Contains(err.Error(), "invalid checksum") {
+			return nil, status.Error(codes.Unauthenticated, "Incorrect password")
+		}
+		return nil, status.Errorf(codes.Internal, "Could not initialize wallet: %v", err)
 	}
 	return s.sendAuthResponse()
 }
diff --git a/validator/rpc/wallet.go b/validator/rpc/wallet.go
index d3f43b5e8a78..71818b7adb44 100644
--- a/validator/rpc/wallet.go
+++ b/validator/rpc/wallet.go
@@ -165,6 +165,9 @@ func (s *Server) ChangePassword(ctx context.Context, req *pb.ChangePasswordReque
 			return nil, status.Error(codes.FailedPrecondition, "Not a valid direct keymanager")
 		}
 		s.wallet.SetPassword(req.Password)
+		if err := s.wallet.SaveHashedPassword(ctx); err != nil {
+			return nil, status.Errorf(codes.Internal, "Could not save hashed password: %v", err)
+		}
 		if err := km.RefreshWalletPassword(ctx); err != nil {
 			return nil, status.Errorf(codes.Internal, "Could not refresh wallet password: %v", err)
 		}
@@ -174,6 +177,9 @@ func (s *Server) ChangePassword(ctx context.Context, req *pb.ChangePasswordReque
 			return nil, status.Error(codes.FailedPrecondition, "Not a valid derived keymanager")
 		}
 		s.wallet.SetPassword(req.Password)
+		if err := s.wallet.SaveHashedPassword(ctx); err != nil {
+			return nil, status.Errorf(codes.Internal, "Could not save hashed password: %v", err)
+		}
 		if err := km.RefreshWalletPassword(ctx); err != nil {
 			return nil, status.Errorf(codes.Internal, "Could not refresh wallet password: %v", err)
 		}