diff --git a/ip.go b/ip.go
index 155cf4991..497c6a78a 100644
--- a/ip.go
+++ b/ip.go
@@ -225,11 +225,11 @@ func extractIP(req *http.Request) string {
 func ExtractIPFromRealIPHeader(options ...TrustOption) IPExtractor {
 	checker := newIPChecker(options)
 	return func(req *http.Request) string {
-		realIP := req.Header.Get(HeaderXRealIP)
 		remoteIP := extractIP(req)
+		realIP := req.Header.Get(HeaderXRealIP)
+		realIP = strings.TrimPrefix(realIP, "[")
+		realIP = strings.TrimSuffix(realIP, "]")
 		if checker.trust(remoteIP) && realIP != "" {
-			realIP = strings.TrimPrefix(realIP, "[")
-			realIP = strings.TrimSuffix(realIP, "]")
 			if ip := net.ParseIP(realIP); ip != nil {
 				return realIP
 			}