diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile
index 3e3b7ac3a6d99..124256378b287 100644
--- a/ci/docker/ubuntu-20.04-cpp.dockerfile
+++ b/ci/docker/ubuntu-20.04-cpp.dockerfile
@@ -101,6 +101,7 @@ RUN apt-get update -y -q && \
         libutf8proc-dev \
         libxml2-dev \
         libzstd-dev \
+        lld \
         make \
         ninja-build \
         nlohmann-json3-dev \
@@ -164,6 +165,7 @@ ENV absl_SOURCE=BUNDLED \
     ARROW_SUBSTRAIT=ON \
     ARROW_USE_ASAN=OFF \
     ARROW_USE_CCACHE=ON \
+    ARROW_USE_LLD=ON \
     ARROW_USE_UBSAN=OFF \
     ARROW_WITH_BROTLI=ON \
     ARROW_WITH_BZ2=ON \
diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh
index 52c89acb9a76a..e28ceae8801f0 100755
--- a/ci/scripts/cpp_build.sh
+++ b/ci/scripts/cpp_build.sh
@@ -166,6 +166,7 @@ else
     -DARROW_USE_CCACHE=${ARROW_USE_CCACHE:-ON} \
     -DARROW_USE_GLOG=${ARROW_USE_GLOG:-OFF} \
     -DARROW_USE_LD_GOLD=${ARROW_USE_LD_GOLD:-OFF} \
+    -DARROW_USE_LLD=${ARROW_USE_LLD:-OFF} \
     -DARROW_USE_MOLD=${ARROW_USE_MOLD:-OFF} \
     -DARROW_USE_PRECOMPILED_HEADERS=${ARROW_USE_PRECOMPILED_HEADERS:-OFF} \
     -DARROW_USE_STATIC_CRT=${ARROW_USE_STATIC_CRT:-OFF} \
diff --git a/cpp/cmake_modules/DefineOptions.cmake b/cpp/cmake_modules/DefineOptions.cmake
index 5b8bcb3ac6965..dc0e5da63adb7 100644
--- a/cpp/cmake_modules/DefineOptions.cmake
+++ b/cpp/cmake_modules/DefineOptions.cmake
@@ -170,6 +170,8 @@ takes precedence over ccache if a storage backend is configured" ON)
 
   define_option(ARROW_USE_LD_GOLD "Use ld.gold for linking on Linux (if available)" OFF)
 
+  define_option(ARROW_USE_LLD "Use the LLVM lld for linking (if available)" OFF)
+
   define_option(ARROW_USE_MOLD "Use mold for linking on Linux (if available)" OFF)
 
   define_option(ARROW_USE_PRECOMPILED_HEADERS "Use precompiled headers when compiling"
diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index 1d709fe98d7fe..3e580dfc109cf 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -663,6 +663,34 @@ if(NOT WIN32 AND NOT APPLE)
   endif()
 endif()
 
+if(ARROW_USE_LLD)
+  find_program(LD_LLD ld.lld)
+  if(LD_LLD)
+    unset(LLD_LINKER_FLAGS)
+    if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+      if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "9.1.0")
+        set(LLD_LINKER_FLAGS "-fuse-ld=lld")
+      else()
+        message(STATUS "Need GCC 9.1.0 or later to use LLD linker: ${CMAKE_CXX_COMPILER_VERSION}"
+        )
+      endif()
+    elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+      set(LLD_LINKER_FLAGS "--ld-path=${LD_LLD}")
+    else()
+      message(STATUS "Using the default linker because compiler doesn't support LLD: ${CMAKE_CXX_COMPILER_ID}"
+      )
+    endif()
+    if(LLD_LINKER_FLAGS)
+      message(STATUS "Using optional LLVM LLD linker")
+      string(APPEND CMAKE_EXE_LINKER_FLAGS " ${LLD_LINKER_FLAGS}")
+      string(APPEND CMAKE_MODULE_LINKER_FLAGS " ${LLD_LINKER_FLAGS}")
+      string(APPEND CMAKE_SHARED_LINKER_FLAGS " ${LLD_LINKER_FLAGS}")
+    else()
+      message(STATUS "Using the default linker because the LLD isn't supported")
+    endif()
+  endif()
+endif()
+
 # compiler flags for different build types (run 'cmake -DCMAKE_BUILD_TYPE=<type> .')
 # For all builds:
 # For CMAKE_BUILD_TYPE=Debug