diff --git a/src/transformers/adapters/trainer.py b/src/transformers/adapters/trainer.py
index 3b8096ee74..9a69f8c233 100644
--- a/src/transformers/adapters/trainer.py
+++ b/src/transformers/adapters/trainer.py
@@ -78,15 +78,15 @@ def __init__(
                 or isinstance(self.model.active_adapters, AdapterCompositionBlock)
                 and any([isinstance(child, Fuse) for child in self.model.active_adapters.children])
             )
-        if model.active_adapters is None:
+        if self.model.active_adapters is None:
             raise ValueError(
                 "Expected a model with an active adapter setup."
                 "If you want to fully finetune the model use the Trainer class."
             )
-        if (self.label_names is None or len(self.label_names) < 1) and model.active_head is not None:
+        if (self.label_names is None or len(self.label_names) < 1) and self.model.active_head is not None:
             all_label_names = set()
-            for head in model._active_heads:
-                all_label_names |= set(model.heads[head].get_label_names())
+            for head in self.model._active_heads:
+                all_label_names |= set(self.model.heads[head].get_label_names())
             self.label_names = list(all_label_names)
 
     def create_optimizer(self):
diff --git a/tests_adapters/test_adapter_trainer.py b/tests_adapters/test_adapter_trainer.py
index 5849bc7bfe..efcb9fe65a 100644
--- a/tests_adapters/test_adapter_trainer.py
+++ b/tests_adapters/test_adapter_trainer.py
@@ -17,7 +17,7 @@
 )
 from transformers.adapters.composition import Fuse, Stack
 from transformers.adapters.trainer import AdapterTrainer, logger
-from transformers.testing_utils import slow
+from transformers.testing_utils import slow, require_ray
 
 
 class TestAdapterTrainer(unittest.TestCase):
@@ -364,6 +364,55 @@ def test_general(self):
             self.assertEqual("task", model.active_head)
             self.assertEqual(Stack("task"), model.active_adapters)
 
+    @require_ray
+    def test_hyperparameter_search_works_with_AdapterTrainer(self):
+        tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
+        data_args = GlueDataTrainingArguments(
+            task_name="mrpc", data_dir="./tests/fixtures/tests_samples/MRPC", overwrite_cache=True
+        )
+        train_dataset = GlueDataset(data_args, tokenizer=tokenizer, mode="train")
+        eval_dataset = train_dataset
+
+        def hp_space(params):
+            from ray import tune
+            return {
+                "learning_rate": tune.choice([0.1, 0.2]),
+            }
+
+        def model_init(trail=None):
+            model = AutoAdapterModel.from_pretrained("bert-base-uncased")
+
+            model.add_classification_head("task", num_labels=3)
+
+            # add the adapters to be fused
+            model.add_adapter("task")
+            model.add_adapter("additional_adapter")
+
+            model.train_adapter("task")
+            return model
+
+        with TemporaryDirectory() as tempdir:
+            training_args = TrainingArguments(
+                output_dir=tempdir,
+                do_train=True,
+                learning_rate=0.1,
+                logging_steps=1,
+                max_steps=1,
+                save_steps=1,
+                remove_unused_columns=False,
+            )
+            trainer = AdapterTrainer(
+                model=None,
+                model_init=model_init,
+                args=training_args,
+                train_dataset=train_dataset,
+                eval_dataset=eval_dataset
+            )
+
+            trainer.hyperparameter_search(
+                direction="minimize", hp_space=hp_space, backend="ray", n_trials=2
+            )
+
 
 if __name__ == "__main__":
     unittest.main()