diff --git a/.gitmodules b/.gitmodules
index 6b14c3524f7e..0513981e5886 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -4,9 +4,6 @@
 [submodule "3rdparty/rang"]
 	path = 3rdparty/rang
 	url = https://github.com/agauniyal/rang.git
-[submodule "3rdparty/libbacktrace"]
-	path = ffi/3rdparty/libbacktrace
-	url = https://github.com/ianlancetaylor/libbacktrace
 [submodule "3rdparty/cutlass"]
 	path = 3rdparty/cutlass
 	url = https://github.com/NVIDIA/cutlass.git
@@ -25,9 +22,6 @@
 [submodule "3rdparty/zlib"]
 	path = 3rdparty/zlib
 	url = https://github.com/madler/zlib.git
-[submodule "ffi/3rdparty/dlpack"]
-	path = ffi/3rdparty/dlpack
-	url = https://github.com/dmlc/dlpack.git
 [submodule "3rdparty/tvm-ffi"]
 	path = 3rdparty/tvm-ffi
 	url = https://github.com/apache/tvm-ffi
diff --git a/Makefile b/Makefile
index 4fdbc7df8448..8ebc28412313 100644
--- a/Makefile
+++ b/Makefile
@@ -37,7 +37,7 @@ TVM_BUILD_PATH := $(abspath $(TVM_BUILD_PATH))
 # Allow environment variables for 3rd-party libraries, default to
 # packaged version.
 DMLC_CORE_PATH ?= $(ROOTDIR)/3rdparty/dmlc-core
-DLPACK_PATH ?= $(ROOTDIR)/ffi/3rdparty/dlpack
+DLPACK_PATH ?= $(ROOTDIR)/3rdparty/tvm-ffi/3rdparty/dlpack
 
 all: $(addsuffix /all,$(TVM_BUILD_PATH))
 
diff --git a/ffi/3rdparty/dlpack b/ffi/3rdparty/dlpack
deleted file mode 160000
index 3ea601bb4130..000000000000
--- a/ffi/3rdparty/dlpack
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 3ea601bb413074c49a77c4ce3218bc08f8c4703c
diff --git a/ffi/3rdparty/libbacktrace b/ffi/3rdparty/libbacktrace
deleted file mode 160000
index 793921876c98..000000000000
--- a/ffi/3rdparty/libbacktrace
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 793921876c981ce49759114d7bb89bb89b2d3a2d