diff --git a/docker/Dockerfile.ci_cpu b/docker/Dockerfile.ci_cpu
index 42067b2cf38b4..305fec7ccba7b 100644
--- a/docker/Dockerfile.ci_cpu
+++ b/docker/Dockerfile.ci_cpu
@@ -78,3 +78,7 @@ RUN bash /install/ubuntu_install_tensorflow.sh
 # Arm(R) Compute Library
 COPY install/ubuntu_install_arm_compute_lib.sh /install/ubuntu_install_arm_compute_lib.sh
 RUN bash /install/ubuntu_install_arm_compute_lib.sh
+
+# Caffe deps
+COPY install/ubuntu_install_caffe.sh /install/ubuntu_install_caffe.sh
+RUN bash /install/ubuntu_install_caffe.sh
diff --git a/docker/install/ubuntu_install_caffe.sh b/docker/install/ubuntu_install_caffe.sh
new file mode 100644
index 0000000000000..21f7d09b1bf3f
--- /dev/null
+++ b/docker/install/ubuntu_install_caffe.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+set -u
+set -o pipefail
+
+apt-get update --fix-missing
+
+# The precompiled caffe dependents on tzdata.
+# While installing tzdata in docker, we need set the time zone manually,
+# which will cause the container to hang during installation.
+# So in order to avoid manually selecting the time zone, set as following:
+export DEBIAN_FRONTEND=noninteractive
+apt-get install -y tzdata
+
+apt-get install caffe-cpu -y
+
+pip3 install --upgrade scikit-image