diff --git a/dev/release/post-09-docs.sh b/dev/release/post-09-docs.sh
index cc0edc06ac2b6..ad74dbce8d0a6 100755
--- a/dev/release/post-09-docs.sh
+++ b/dev/release/post-09-docs.sh
@@ -24,15 +24,25 @@ SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 ARROW_DIR="${SOURCE_DIR}/../.."
 ARROW_SITE_DIR="${ARROW_DIR}/../arrow-site"
 
-if [ "$#" -ne 1 ]; then
-  echo "Usage: $0 <version>"
+if [ "$#" -ne 2  ]; then
+  echo "Usage: $0 <version> <previous_version>"
   exit 1
 fi
 
 version=$1
+previous_version=$2
 release_tag="apache-arrow-${version}"
 branch_name=release-docs-${version}
 
+case "${version}" in
+  *.0.0)
+    is_major_release=yes
+    ;;
+  *)
+    is_major_release=no
+    ;;
+esac
+
 pushd "${ARROW_SITE_DIR}"
 source "${SOURCE_DIR}/git-vars.sh"
 git fetch --all --prune --tags --force -j$(nproc)
@@ -44,10 +54,22 @@ git checkout -b asf-site origin/asf-site
 git rebase apache/asf-site
 git branch -D ${branch_name} || :
 git checkout -b ${branch_name}
+# list and remove previous versioned docs
 versioned_paths=()
 for versioned_path in docs/*.0/; do
   versioned_paths+=(${versioned_path})
+  rm -rf ${versioned_path}
 done
+# add to list and remove dev docs
+versioned_paths+=("docs/dev/")
+rm -rf docs/dev/
+if [ "$is_major_release" = "yes" ] ; then
+  # copy the current stable docs to temporary directory
+  # (remove java reference to reduce size)
+  rm -rf docs/java/reference/
+  cp -r docs/ docs_temp/
+fi
+# delete current stable docs and restore all previous versioned docs
 rm -rf docs/*
 git checkout "${versioned_paths[@]}"
 curl \
@@ -58,8 +80,12 @@ curl \
 tar xvf docs.tar.gz
 rm -f docs.tar.gz
 git checkout docs/c_glib/index.html
+if [ "$is_major_release" = "yes" ] ; then
+  mv docs_temp docs/${previous_version}
+fi
 git add docs
 git commit -m "[Website] Update documentations for ${version}"
+git clean -d -f -x
 popd
 
 : ${PUSH:=1}