diff --git a/.circleci/config.yml b/.circleci/config.yml
index 9a2148c3c8..f3834d98c5 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -112,6 +112,9 @@ install_linux_dep: &install_linux_dep
         # install from github to get latest; install iopath first since fvcore depends on it
         pip install --progress-bar off -U 'git+https://github.com/facebookresearch/iopath'
         pip install --progress-bar off -U 'git+https://github.com/facebookresearch/fvcore'
+        # pytorch ships proto files which are incompatible with protobuf >=4,
+        # see https://github.com/pytorch/pytorch/issues/78362
+        pip install --progress-bar off protobuf~=3.20.3
         # Don't use pytest-xdist: cuda tests are unstable under multi-process workers.
         # Don't use opencv 4.7.0.68: https://github.com/opencv/opencv-python/issues/765
         pip install --progress-bar off ninja opencv-python-headless!=4.7.0.68 pytest tensorboard pycocotools onnx
diff --git a/setup.py b/setup.py
index b51307a625..b82e13db1b 100644
--- a/setup.py
+++ b/setup.py
@@ -190,6 +190,7 @@ def get_model_zoo_configs() -> List[str]:
         "hydra-core>=1.1",
         "black",
         "packaging",
+        "protobuf~=3.20.3",  # pytorch ships proto files which are incompatible with protobuf >=4 (see https://github.com/pytorch/pytorch/issues/78362)
         # NOTE: When adding new dependencies, if it is required at import time (in addition
         # to runtime), it probably needs to appear in docs/requirements.txt, or as a mock
         # in docs/conf.py