diff --git a/clippy_lints/src/arithmetic.rs b/clippy_lints/src/arithmetic.rs
index fd252b9ee109..a138c9d3545c 100644
--- a/clippy_lints/src/arithmetic.rs
+++ b/clippy_lints/src/arithmetic.rs
@@ -81,11 +81,12 @@ impl<'a, 'tcx> LateLintPass<'a, 'tcx> for Arithmetic {
                     | hir::BinOpKind::Gt => return,
                     _ => (),
                 }
+
                 let (l_ty, r_ty) = (cx.tables.expr_ty(l), cx.tables.expr_ty(r));
-                if l_ty.is_integral() && r_ty.is_integral() {
+                if l_ty.peel_refs().is_integral() && r_ty.peel_refs().is_integral() {
                     span_lint(cx, INTEGER_ARITHMETIC, expr.span, "integer arithmetic detected");
                     self.expr_span = Some(expr.span);
-                } else if l_ty.is_floating_point() && r_ty.is_floating_point() {
+                } else if l_ty.peel_refs().is_floating_point() && r_ty.peel_refs().is_floating_point() {
                     span_lint(cx, FLOAT_ARITHMETIC, expr.span, "floating-point arithmetic detected");
                     self.expr_span = Some(expr.span);
                 }
diff --git a/tests/ui/arithmetic.rs b/tests/ui/arithmetic.rs
index dc11cbd98deb..38892f6b14de 100644
--- a/tests/ui/arithmetic.rs
+++ b/tests/ui/arithmetic.rs
@@ -4,7 +4,9 @@
     clippy::shadow_reuse,
     clippy::shadow_unrelated,
     clippy::no_effect,
-    clippy::unnecessary_operation
+    clippy::unnecessary_operation,
+    clippy::op_ref,
+    clippy::trivially_copy_pass_by_ref
 )]
 
 #[rustfmt::skip]
@@ -90,4 +92,55 @@ fn main() {
             1 + 1
         };
     }
+
+
+}
+
+// warn on references as well! (#5328)
+pub fn int_arith_ref() {
+    3 + &1;
+    &3 + 1;
+    &3 + &1;
+}
+
+pub fn foo(x: &i32) -> i32 {
+    let a = 5;
+    a + x
+}
+
+pub fn bar(x: &i32, y: &i32) -> i32 {
+    x + y
+}
+
+pub fn baz(x: i32, y: &i32) -> i32 {
+    x + y
+}
+
+pub fn qux(x: i32, y: i32) -> i32 {
+    (&x + &y)
+}
+
+// also warn about floating point arith with references involved
+
+pub fn float_arith_ref() {
+    3.1_f32 + &1.2_f32;
+    &3.4_f32 + 1.5_f32;
+    &3.5_f32 + &1.3_f32;
+}
+
+pub fn float_foo(f: &f32) -> f32 {
+    let a = 5.1;
+    a + f
+}
+
+pub fn float_bar(f1: &f32, f2: &f32) -> f32 {
+    f1 + f2
+}
+
+pub fn float_baz(f1: f32, f2: &f32) -> f32 {
+    f1 + f2
+}
+
+pub fn float_qux(f1: f32, f2: f32) -> f32 {
+    (&f1 + &f2)
 }
diff --git a/tests/ui/arithmetic.stderr b/tests/ui/arithmetic.stderr
index d999b69d7cbb..5de2eecf128d 100644
--- a/tests/ui/arithmetic.stderr
+++ b/tests/ui/arithmetic.stderr
@@ -1,5 +1,5 @@
 error: integer arithmetic detected
-  --> $DIR/arithmetic.rs:13:5
+  --> $DIR/arithmetic.rs:15:5
    |
 LL |     1 + i;
    |     ^^^^^
@@ -7,62 +7,62 @@ LL |     1 + i;
    = note: `-D clippy::integer-arithmetic` implied by `-D warnings`
 
 error: integer arithmetic detected
-  --> $DIR/arithmetic.rs:14:5
+  --> $DIR/arithmetic.rs:16:5
    |
 LL |     i * 2;
    |     ^^^^^
 
 error: integer arithmetic detected
-  --> $DIR/arithmetic.rs:15:5
+  --> $DIR/arithmetic.rs:17:5
    |
 LL | /     1 %
 LL | |     i / 2; // no error, this is part of the expression in the preceding line
    | |_________^
 
 error: integer arithmetic detected
-  --> $DIR/arithmetic.rs:17:5
+  --> $DIR/arithmetic.rs:19:5
    |
 LL |     i - 2 + 2 - i;
    |     ^^^^^^^^^^^^^
 
 error: integer arithmetic detected
-  --> $DIR/arithmetic.rs:18:5
+  --> $DIR/arithmetic.rs:20:5
    |
 LL |     -i;
    |     ^^
 
 error: integer arithmetic detected
-  --> $DIR/arithmetic.rs:30:5
+  --> $DIR/arithmetic.rs:32:5
    |
 LL |     i += 1;
    |     ^^^^^^
 
 error: integer arithmetic detected
-  --> $DIR/arithmetic.rs:31:5
+  --> $DIR/arithmetic.rs:33:5
    |
 LL |     i -= 1;
    |     ^^^^^^
 
 error: integer arithmetic detected
-  --> $DIR/arithmetic.rs:32:5
+  --> $DIR/arithmetic.rs:34:5
    |
 LL |     i *= 2;
    |     ^^^^^^
 
 error: integer arithmetic detected
-  --> $DIR/arithmetic.rs:33:5
+  --> $DIR/arithmetic.rs:35:5
    |
 LL |     i /= 2;
    |     ^^^^^^
 
 error: integer arithmetic detected
-  --> $DIR/arithmetic.rs:34:5
+  --> $DIR/arithmetic.rs:36:5
    |
 LL |     i %= 2;
    |     ^^^^^^
 
 error: floating-point arithmetic detected
-  --> $DIR/arithmetic.rs:45:5
+  --> $DIR/arithmetic.rs:47:5
    |
 LL |     f * 2.0;
    |     ^^^^^^^
@@ -70,58 +70,142 @@ LL |     f * 2.0;
    = note: `-D clippy::float-arithmetic` implied by `-D warnings`
 
 error: floating-point arithmetic detected
-  --> $DIR/arithmetic.rs:47:5
+  --> $DIR/arithmetic.rs:49:5
    |
 LL |     1.0 + f;
    |     ^^^^^^^
 
 error: floating-point arithmetic detected
-  --> $DIR/arithmetic.rs:48:5
+  --> $DIR/arithmetic.rs:50:5
    |
 LL |     f * 2.0;
    |     ^^^^^^^
 
 error: floating-point arithmetic detected
-  --> $DIR/arithmetic.rs:49:5
+  --> $DIR/arithmetic.rs:51:5
    |
 LL |     f / 2.0;
    |     ^^^^^^^
 
 error: floating-point arithmetic detected
-  --> $DIR/arithmetic.rs:50:5
+  --> $DIR/arithmetic.rs:52:5
    |
 LL |     f - 2.0 * 4.2;
    |     ^^^^^^^^^^^^^
 
 error: floating-point arithmetic detected
-  --> $DIR/arithmetic.rs:51:5
+  --> $DIR/arithmetic.rs:53:5
    |
 LL |     -f;
    |     ^^
 
 error: floating-point arithmetic detected
-  --> $DIR/arithmetic.rs:53:5
+  --> $DIR/arithmetic.rs:55:5
    |
 LL |     f += 1.0;
    |     ^^^^^^^^
 
 error: floating-point arithmetic detected
-  --> $DIR/arithmetic.rs:54:5
+  --> $DIR/arithmetic.rs:56:5
    |
 LL |     f -= 1.0;
    |     ^^^^^^^^
 
 error: floating-point arithmetic detected
-  --> $DIR/arithmetic.rs:55:5
+  --> $DIR/arithmetic.rs:57:5
    |
 LL |     f *= 2.0;
    |     ^^^^^^^^
 
 error: floating-point arithmetic detected
-  --> $DIR/arithmetic.rs:56:5
+  --> $DIR/arithmetic.rs:58:5
    |
 LL |     f /= 2.0;
    |     ^^^^^^^^
 
-error: aborting due to 20 previous errors
+error: integer arithmetic detected
+  --> $DIR/arithmetic.rs:101:5
+   |
+LL |     3 + &1;
+   |     ^^^^^^
+
+error: integer arithmetic detected
+  --> $DIR/arithmetic.rs:102:5
+   |
+LL |     &3 + 1;
+   |     ^^^^^^
+
+error: integer arithmetic detected
+  --> $DIR/arithmetic.rs:103:5
+   |
+LL |     &3 + &1;
+   |     ^^^^^^^
+
+error: integer arithmetic detected
+  --> $DIR/arithmetic.rs:108:5
+   |
+LL |     a + x
+   |     ^^^^^
+
+error: integer arithmetic detected
+  --> $DIR/arithmetic.rs:112:5
+   |
+LL |     x + y
+   |     ^^^^^
+
+error: integer arithmetic detected
+  --> $DIR/arithmetic.rs:116:5
+   |
+LL |     x + y
+   |     ^^^^^
+
+error: integer arithmetic detected
+  --> $DIR/arithmetic.rs:120:5
+   |
+LL |     (&x + &y)
+   |     ^^^^^^^^^
+
+error: floating-point arithmetic detected
+  --> $DIR/arithmetic.rs:126:5
+   |
+LL |     3.1_f32 + &1.2_f32;
+   |     ^^^^^^^^^^^^^^^^^^
+
+error: floating-point arithmetic detected
+  --> $DIR/arithmetic.rs:127:5
+   |
+LL |     &3.4_f32 + 1.5_f32;
+   |     ^^^^^^^^^^^^^^^^^^
+
+error: floating-point arithmetic detected
+  --> $DIR/arithmetic.rs:128:5
+   |
+LL |     &3.5_f32 + &1.3_f32;
+   |     ^^^^^^^^^^^^^^^^^^^
+
+error: floating-point arithmetic detected
+  --> $DIR/arithmetic.rs:133:5
+   |
+LL |     a + f
+   |     ^^^^^
+
+error: floating-point arithmetic detected
+  --> $DIR/arithmetic.rs:137:5
+   |
+LL |     f1 + f2
+   |     ^^^^^^^
+
+error: floating-point arithmetic detected
+  --> $DIR/arithmetic.rs:141:5
+   |
+LL |     f1 + f2
+   |     ^^^^^^^
+
+error: floating-point arithmetic detected
+  --> $DIR/arithmetic.rs:145:5
+   |
+LL |     (&f1 + &f2)
+   |     ^^^^^^^^^^^
+
+error: aborting due to 34 previous errors