Fixes after sync

examples/pytorch/image-classification/run_image_classification.py

@@ -37,15 +37,18 @@
 import transformers
 from transformers import (
  MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
+ AdapterArguments,
+ AdapterConfig,
+ AdapterTrainer,
+ AutoAdapterModel,
  AutoConfig,
  AutoFeatureExtractor,
- AutoModelForImageClassification,
  HfArgumentParser,
  Trainer,
  TrainingArguments,
 )
 from transformers.trainer_utils import get_last_checkpoint
-from transformers.utils import check_min_version, send_example_telemetry
+from transformers.utils import check_min_version
 from transformers.utils.versions import require_version
 
 
@@ -167,17 +170,15 @@ def main():
  # or by passing the --help flag to this script.
  # We now keep distinct sets of args, for a cleaner separation of concerns.
 
- parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
+ parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments, AdapterArguments))
  if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
  # If we pass only one argument to the script and it's the path to a json file,
  # let's parse it to get our arguments.
- model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
+ model_args, data_args, training_args, adapter_args = parser.parse_json_file(
+ json_file=os.path.abspath(sys.argv[1])
+ )
  else:
- model_args, data_args, training_args = parser.parse_args_into_dataclasses()
-
- # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
- # information sent is the one passed as arguments along with your Python/PyTorch versions.
- send_example_telemetry("run_image_classification", model_args, data_args)
+ model_args, data_args, training_args, adapter_args = parser.parse_args_into_dataclasses()
 
  # Setup logging
  logging.basicConfig(
@@ -270,7 +271,7 @@ def compute_metrics(p):
  revision=model_args.model_revision,
  use_auth_token=True if model_args.use_auth_token else None,
  )
- model = AutoModelForImageClassification.from_pretrained(
+ model = AutoAdapterModel.from_pretrained(
  model_args.model_name_or_path,
  from_tf=bool(".ckpt" in model_args.model_name_or_path),
  config=config,
@@ -279,13 +280,50 @@ def compute_metrics(p):
  use_auth_token=True if model_args.use_auth_token else None,
  ignore_mismatched_sizes=model_args.ignore_mismatched_sizes,
  )
+ model.add_image_classification_head(
+ data_args.dataset_name or "img_clf",
+ num_labels=len(labels),
+ id2label=id2label,
+ )
  feature_extractor = AutoFeatureExtractor.from_pretrained(
  model_args.feature_extractor_name or model_args.model_name_or_path,
  cache_dir=model_args.cache_dir,
  revision=model_args.model_revision,
  use_auth_token=True if model_args.use_auth_token else None,
  )
 
+ # Setup adapters
+ if adapter_args.train_adapter:
+ task_name = data_args.dataset_name or "img_clf"
+ # check if adapter already exists, otherwise add it
+ if task_name not in model.config.adapters:
+ # resolve the adapter config
+ adapter_config = AdapterConfig.load(
+ adapter_args.adapter_config,
+ non_linearity=adapter_args.adapter_non_linearity,
+ reduction_factor=adapter_args.adapter_reduction_factor,
+ )
+ # load a pre-trained from Hub if specified
+ if adapter_args.load_adapter:
+ model.load_adapter(
+ adapter_args.load_adapter,
+ config=adapter_config,
+ load_as=task_name,
+ )
+ # otherwise, add a fresh adapter
+ else:
+ model.add_adapter(task_name, config=adapter_config)
+
+ # Freeze all model weights except of those of this adapter
+ model.train_adapter([task_name])
+ # Set the adapters to be used in every forward pass
+ model.set_active_adapters([task_name])
+ else:
+ if adapter_args.load_adapter:
+ raise ValueError(
+ "Adapters can only be loaded in adapters training mode.Use --train_adapter to enable adapter training"
+ )
+
  # Define torchvision transforms to be applied to each image.
  normalize = Normalize(mean=feature_extractor.image_mean, std=feature_extractor.image_std)
  _train_transforms = Compose(
@@ -338,7 +376,8 @@ def val_transforms(example_batch):
  dataset["validation"].set_transform(val_transforms)
 
  # Initalize our trainer
- trainer = Trainer(
+ trainer_class = AdapterTrainer if adapter_args.train_adapter else Trainer
+ trainer = trainer_class(
  model=model,
  args=training_args,
  train_dataset=dataset["train"] if training_args.do_train else None,

examples/pytorch/language-modeling/run_clm.py

@@ -33,22 +33,26 @@
 from datasets import load_dataset, load_metric
 
 import transformers
+import transformers.adapters.composition as ac
 from transformers import (
  CONFIG_MAPPING,
  MODEL_FOR_CAUSAL_LM_MAPPING,
+ AdapterTrainer,
  AutoConfig,
  AutoModelForCausalLM,
  AutoTokenizer,
  HfArgumentParser,
+ MultiLingAdapterArguments,
  Trainer,
  TrainingArguments,
  default_data_collator,
  is_torch_tpu_available,
  set_seed,
 )
+from transformers.adapters.configuration import AdapterConfig
 from transformers.testing_utils import CaptureLogger
 from transformers.trainer_utils import get_last_checkpoint
-from transformers.utils import check_min_version, send_example_telemetry
+from transformers.utils import check_min_version
 from transformers.utils.versions import require_version
 
 
@@ -206,17 +210,16 @@ def main():
  # or by passing the --help flag to this script.
  # We now keep distinct sets of args, for a cleaner separation of concerns.
 
- parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
+ parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments, MultiLingAdapterArguments))
+
  if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
  # If we pass only one argument to the script and it's the path to a json file,
  # let's parse it to get our arguments.
- model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
+ model_args, data_args, training_args, adapter_args = parser.parse_json_file(
+ json_file=os.path.abspath(sys.argv[1])
+ )
  else:
- model_args, data_args, training_args = parser.parse_args_into_dataclasses()
-
- # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
- # information sent is the one passed as arguments along with your Python/PyTorch versions.
- send_example_telemetry("run_clm", model_args, data_args)
+ model_args, data_args, training_args, adapter_args = parser.parse_args_into_dataclasses()
 
  # Setup logging
  logging.basicConfig(
@@ -388,6 +391,56 @@ def main():
 
  model.resize_token_embeddings(len(tokenizer))
 
+ # Setup adapters
+ if adapter_args.train_adapter:
+ task_name = data_args.dataset_name or "clm"
+ # check if adapter already exists, otherwise add it
+ if task_name not in model.config.adapters:
+ # resolve the adapter config
+ adapter_config = AdapterConfig.load(
+ adapter_args.adapter_config,
+ non_linearity=adapter_args.adapter_non_linearity,
+ reduction_factor=adapter_args.adapter_reduction_factor,
+ )
+ # load a pre-trained from Hub if specified
+ if adapter_args.load_adapter:
+ model.load_adapter(
+ adapter_args.load_adapter,
+ config=adapter_config,
+ load_as=task_name,
+ )
+ # otherwise, add a fresh adapter
+ else:
+ model.add_adapter(task_name, config=adapter_config)
+ # optionally load a pre-trained language adapter
+ if adapter_args.load_lang_adapter:
+ # resolve the language adapter config
+ lang_adapter_config = AdapterConfig.load(
+ adapter_args.lang_adapter_config,
+ non_linearity=adapter_args.lang_adapter_non_linearity,
+ reduction_factor=adapter_args.lang_adapter_reduction_factor,
+ )
+ # load the language adapter from Hub
+ lang_adapter_name = model.load_adapter(
+ adapter_args.load_lang_adapter,
+ config=lang_adapter_config,
+ load_as=adapter_args.language,
+ )
+ else:
+ lang_adapter_name = None
+ # Freeze all model weights except of those of this adapter
+ model.train_adapter([task_name])
+ # Set the adapters to be used in every forward pass
+ if lang_adapter_name:
+ model.set_active_adapters(ac.Stack(lang_adapter_name, task_name))
+ else:
+ model.set_active_adapters(task_name)
+ else:
+ if adapter_args.load_adapter or adapter_args.load_lang_adapter:
+ raise ValueError(
+ "Adapters can only be loaded in adapters training mode.Use --train_adapter to enable adapter training"
+ )
+
  # Preprocessing the datasets.
  # First we tokenize all the texts.
  if training_args.do_train:
@@ -503,7 +556,8 @@ def compute_metrics(eval_preds):
  return metric.compute(predictions=preds, references=labels)
 
  # Initialize our Trainer
- trainer = Trainer(
+ trainer_class = AdapterTrainer if adapter_args.train_adapter else Trainer
+ trainer = trainer_class(
  model=model,
  args=training_args,
  train_dataset=train_dataset if training_args.do_train else None,

examples/pytorch/language-modeling/run_mlm.py

@@ -33,21 +33,25 @@
 from datasets import load_dataset, load_metric
 
 import transformers
+import transformers.adapters.composition as ac
 from transformers import (
  CONFIG_MAPPING,
  MODEL_FOR_MASKED_LM_MAPPING,
+ AdapterTrainer,
  AutoConfig,
  AutoModelForMaskedLM,
  AutoTokenizer,
  DataCollatorForLanguageModeling,
  HfArgumentParser,
+ MultiLingAdapterArguments,
  Trainer,
  TrainingArguments,
  is_torch_tpu_available,
  set_seed,
 )
+from transformers.adapters.configuration import AdapterConfig
 from transformers.trainer_utils import get_last_checkpoint
-from transformers.utils import check_min_version, send_example_telemetry
+from transformers.utils import check_min_version
 from transformers.utils.versions import require_version
 
 
@@ -216,17 +220,15 @@ def main():
  # or by passing the --help flag to this script.
  # We now keep distinct sets of args, for a cleaner separation of concerns.
 
- parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
+ parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments, MultiLingAdapterArguments))
  if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
  # If we pass only one argument to the script and it's the path to a json file,
  # let's parse it to get our arguments.
- model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
+ model_args, data_args, training_args, adapter_args = parser.parse_json_file(
+ json_file=os.path.abspath(sys.argv[1])
+ )
  else:
- model_args, data_args, training_args = parser.parse_args_into_dataclasses()
-
- # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
- # information sent is the one passed as arguments along with your Python/PyTorch versions.
- send_example_telemetry("run_mlm", model_args, data_args)
+ model_args, data_args, training_args, adapter_args = parser.parse_args_into_dataclasses()
 
  # Setup logging
  logging.basicConfig(
@@ -390,6 +392,56 @@ def main():
 
  model.resize_token_embeddings(len(tokenizer))
 
+ # Setup adapters
+ if adapter_args.train_adapter:
+ task_name = data_args.dataset_name or "mlm"
+ # check if adapter already exists, otherwise add it
+ if task_name not in model.config.adapters:
+ # resolve the adapter config
+ adapter_config = AdapterConfig.load(
+ adapter_args.adapter_config,
+ non_linearity=adapter_args.adapter_non_linearity,
+ reduction_factor=adapter_args.adapter_reduction_factor,
+ )
+ # load a pre-trained from Hub if specified
+ if adapter_args.load_adapter:
+ model.load_adapter(
+ adapter_args.load_adapter,
+ config=adapter_config,
+ load_as=task_name,
+ )
+ # otherwise, add a fresh adapter
+ else:
+ model.add_adapter(task_name, config=adapter_config)
+ # optionally load a pre-trained language adapter
+ if adapter_args.load_lang_adapter:
+ # resolve the language adapter config
+ lang_adapter_config = AdapterConfig.load(
+ adapter_args.lang_adapter_config,
+ non_linearity=adapter_args.lang_adapter_non_linearity,
+ reduction_factor=adapter_args.lang_adapter_reduction_factor,
+ )
+ # load the language adapter from Hub
+ lang_adapter_name = model.load_adapter(
+ adapter_args.load_lang_adapter,
+ config=lang_adapter_config,
+ load_as=adapter_args.language,
+ )
+ else:
+ lang_adapter_name = None
+ # Freeze all model weights except of those of this adapter
+ model.train_adapter([task_name])
+ # Set the adapters to be used in every forward pass
+ if lang_adapter_name:
+ model.set_active_adapters(ac.Stack(lang_adapter_name, task_name))
+ else:
+ model.set_active_adapters(task_name)
+ else:
+ if adapter_args.load_adapter or adapter_args.load_lang_adapter:
+ raise ValueError(
+ "Adapters can only be loaded in adapters training mode.Use --train_adapter to enable adapter training"
+ )
+
  # Preprocessing the datasets.
  # First we tokenize all the texts.
  if training_args.do_train:
@@ -538,7 +590,8 @@ def compute_metrics(eval_preds):
  )
 
  # Initialize our Trainer
- trainer = Trainer(
+ trainer_class = AdapterTrainer if adapter_args.train_adapter else Trainer
+ trainer = trainer_class(
  model=model,
  args=training_args,
  train_dataset=train_dataset if training_args.do_train else None,