diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml
index 5dc7550d868..ec3eca37afb 100644
--- a/.github/workflows/test-linux-cpu.yml
+++ b/.github/workflows/test-linux-cpu.yml
@@ -54,4 +54,6 @@ jobs:
 
         # Run Tests
         python3 -m torch.utils.collect_env
+        rm -rf torchvision/prototype
+        rm test/*prototype*
         python3 -m pytest --junitxml=test-results/junit.xml -v --durations 20