diff --git a/.gitmodules b/.gitmodules
index 370aae686..4a575a0e8 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,3 +1,6 @@
 [submodule "thirdparty/grpc"]
 	path = thirdparty/grpc
 	url = https://github.com/grpc/grpc
+[submodule "thirdparty/numbuf"]
+	path = thirdparty/numbuf
+	url = https://github.com/amplab/numbuf.git
diff --git a/CMakeLists.txt b/CMakeLists.txt
index fb0e5990c..32dc488ad 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -102,7 +102,6 @@ add_library(pynumbuf STATIC ${CMAKE_SOURCE_DIR}/thirdparty/numbuf/cpp/src/numbuf
                             ${CMAKE_SOURCE_DIR}/thirdparty/numbuf/cpp/src/numbuf/metadata.cc
                             ${CMAKE_SOURCE_DIR}/thirdparty/numbuf/cpp/src/numbuf/dict.cc
                             ${CMAKE_SOURCE_DIR}/thirdparty/numbuf/python/src/pynumbuf/serialize.cc
-                            ${CMAKE_SOURCE_DIR}/thirdparty/numbuf/python/src/pynumbuf/numbuf.cc
                             ${CMAKE_SOURCE_DIR}/thirdparty/numbuf/python/src/pynumbuf/adapters/numpy.cc
                             ${CMAKE_SOURCE_DIR}/thirdparty/numbuf/python/src/pynumbuf/adapters/python.cc)
 target_link_libraries(pynumbuf ${ARROW_LIB} ${PYTHON_LIBRARIES})
diff --git a/thirdparty/download_thirdparty.sh b/thirdparty/download_thirdparty.sh
index 786137d4f..07a63cd0b 100755
--- a/thirdparty/download_thirdparty.sh
+++ b/thirdparty/download_thirdparty.sh
@@ -13,11 +13,6 @@ if [ ! -d arrow ]; then
   cd ..
 fi
 
-if [ ! -d numbuf ]; then
-  echo "Fetching numbuf"
-  git clone https://github.com/amplab/numbuf.git
-fi
-
 git submodule update --init --recursive
 
 # this seems to be neeccessary for building on Mac OS X
diff --git a/thirdparty/numbuf b/thirdparty/numbuf
new file mode 160000
index 000000000..6c5a58b56
--- /dev/null
+++ b/thirdparty/numbuf
@@ -0,0 +1 @@
+Subproject commit 6c5a58b560a72c5a2dfdd92901b825c219a9052d