diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs
index 60f5b79de1033..5e59475a7bdfc 100644
--- a/compiler/rustc_hir/src/hir.rs
+++ b/compiler/rustc_hir/src/hir.rs
@@ -1787,6 +1787,14 @@ impl Expr<'_> {
         expr
     }
 
+    pub fn peel_borrows(&self) -> &Self {
+        let mut expr = self;
+        while let ExprKind::AddrOf(.., inner) = &expr.kind {
+            expr = inner;
+        }
+        expr
+    }
+
     pub fn can_have_side_effects(&self) -> bool {
         match self.peel_drop_temps().kind {
             ExprKind::Path(_) | ExprKind::Lit(_) => false,
diff --git a/compiler/rustc_hir_typeck/src/cast.rs b/compiler/rustc_hir_typeck/src/cast.rs
index b312a3d30af08..712f9b87aed0a 100644
--- a/compiler/rustc_hir_typeck/src/cast.rs
+++ b/compiler/rustc_hir_typeck/src/cast.rs
@@ -31,6 +31,7 @@
 use super::FnCtxt;
 
 use crate::type_error_struct;
+use hir::ExprKind;
 use rustc_errors::{
     struct_span_err, Applicability, DelayDm, Diagnostic, DiagnosticBuilder, ErrorGuaranteed,
 };
@@ -237,12 +238,23 @@ impl<'a, 'tcx> CastCheck<'tcx> {
                     fcx,
                 );
 
-                err.span_suggestion_verbose(
-                    self.expr_span.shrink_to_lo(),
-                    "dereference the expression",
-                    "*",
-                    Applicability::MachineApplicable,
-                );
+                if matches!(self.expr.kind, ExprKind::AddrOf(..)) {
+                    // get just the borrow part of the expression
+                    let span = self.expr_span.with_hi(self.expr.peel_borrows().span.lo());
+                    err.span_suggestion_verbose(
+                        span,
+                        "remove the unneeded borrow",
+                        "",
+                        Applicability::MachineApplicable,
+                    );
+                } else {
+                    err.span_suggestion_verbose(
+                        self.expr_span.shrink_to_lo(),
+                        "dereference the expression",
+                        "*",
+                        Applicability::MachineApplicable,
+                    );
+                }
 
                 err.emit();
             }
diff --git a/tests/ui/error-codes/E0606.rs b/tests/ui/error-codes/E0606.rs
index cb0d8cfc31e23..6f6c6513846cf 100644
--- a/tests/ui/error-codes/E0606.rs
+++ b/tests/ui/error-codes/E0606.rs
@@ -1,3 +1,4 @@
 fn main() {
-    &0u8 as u8; //~ ERROR E0606
+    let x = &(&0u8 as u8); //~ ERROR E0606
+    x as u8; //~ casting `&u8` as `u8` is invalid [E0606]
 }
diff --git a/tests/ui/error-codes/E0606.stderr b/tests/ui/error-codes/E0606.stderr
index 586b1f2fd5472..2492eb299cc55 100644
--- a/tests/ui/error-codes/E0606.stderr
+++ b/tests/ui/error-codes/E0606.stderr
@@ -1,14 +1,26 @@
 error[E0606]: casting `&u8` as `u8` is invalid
-  --> $DIR/E0606.rs:2:5
+  --> $DIR/E0606.rs:2:14
    |
-LL |     &0u8 as u8;
-   |     ^^^^^^^^^^
+LL |     let x = &(&0u8 as u8);
+   |              ^^^^^^^^^^^^
+   |
+help: remove the unneeded borrow
+   |
+LL -     let x = &(&0u8 as u8);
+LL +     let x = &(0u8 as u8);
+   |
+
+error[E0606]: casting `&u8` as `u8` is invalid
+  --> $DIR/E0606.rs:3:5
+   |
+LL |     x as u8;
+   |     ^^^^^^^
    |
 help: dereference the expression
    |
-LL |     *&0u8 as u8;
+LL |     *x as u8;
    |     +
 
-error: aborting due to previous error
+error: aborting due to 2 previous errors
 
 For more information about this error, try `rustc --explain E0606`.