diff --git a/setup.py b/setup.py
index e6f442e5ec23e3..e974ff9a2b2b9a 100644
--- a/setup.py
+++ b/setup.py
@@ -145,7 +145,7 @@
     "regex!=2019.12.17",
     "requests",
     "rjieba",
-    "rouge-score",
+    "rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1",
     "sacrebleu>=1.4.12,<2.0.0",
     "sacremoses",
     "sagemaker>=2.31.0",
diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py
index fec2fc9203240d..c8f0f18793c0a5 100644
--- a/src/transformers/dependency_versions_table.py
+++ b/src/transformers/dependency_versions_table.py
@@ -51,7 +51,7 @@
     "regex": "regex!=2019.12.17",
     "requests": "requests",
     "rjieba": "rjieba",
-    "rouge-score": "rouge-score",
+    "rouge-score": "rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1",
     "sacrebleu": "sacrebleu>=1.4.12,<2.0.0",
     "sacremoses": "sacremoses",
     "sagemaker": "sagemaker>=2.31.0",