From 8e924fb9c546cc6f29a0dae58e96df2eb46ddb21 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 7 Nov 2023 10:35:09 +0000 Subject: [PATCH 1/4] make safe serialization the default one --- src/peft/peft_model.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index 64e70f2ba7..6dedcde210 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -155,7 +155,7 @@ def peft_config(self, value: Dict[str, PeftConfig]): def save_pretrained( self, save_directory: str, - safe_serialization: bool = False, + safe_serialization: bool = True, selected_adapters: Optional[List[str]] = None, **kwargs: Any, ): @@ -168,6 +168,8 @@ def save_pretrained( save_directory (`str`): Directory where the adapter model and configuration files will be saved (will be created if it does not exist). + safe_serialization (`bool`, *optional*): + Whether to save the adapter files in safetensors format. kwargs (additional keyword arguments, *optional*): Additional keyword arguments passed along to the `push_to_hub` method. """ From 43c8f6906db44392da2e352780eb6f3f3b39d6c9 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 7 Nov 2023 10:47:42 +0000 Subject: [PATCH 2/4] adapt tests --- tests/test_custom_models.py | 4 ++ tests/test_decoder_models.py | 4 ++ tests/test_encoder_decoder_models.py | 4 ++ tests/test_gpu_examples.py | 2 +- tests/testing_common.py | 77 ++++++++++++++++++++++++---- 5 files changed, 80 insertions(+), 11 deletions(-) diff --git a/tests/test_custom_models.py b/tests/test_custom_models.py index 9bd4dec9b6..13a98b6d32 100644 --- a/tests/test_custom_models.py +++ b/tests/test_custom_models.py @@ -408,6 +408,10 @@ def test_prepare_for_training_parametrized(self, test_name, model_id, config_cls def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs): self._test_save_pretrained(model_id, config_cls, config_kwargs) + @parameterized.expand(TEST_CASES) + def test_save_pretrained_regression(self, test_name, model_id, config_cls, config_kwargs): + self._test_save_pretrained_regression(model_id, config_cls, config_kwargs) + @parameterized.expand(TEST_CASES) def test_from_pretrained_config_construction(self, test_name, model_id, config_cls, config_kwargs): self._test_from_pretrained_config_construction(model_id, config_cls, config_kwargs) diff --git a/tests/test_decoder_models.py b/tests/test_decoder_models.py index ea30a8183c..622d978e60 100644 --- a/tests/test_decoder_models.py +++ b/tests/test_decoder_models.py @@ -80,6 +80,10 @@ def test_prepare_for_training_parametrized(self, test_name, model_id, config_cls def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs): self._test_save_pretrained(model_id, config_cls, config_kwargs) + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) + def test_save_pretrained_regression(self, test_name, model_id, config_cls, config_kwargs): + self._test_save_pretrained_regression(model_id, config_cls, config_kwargs) + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) def test_save_pretrained_selected_adapters(self, test_name, model_id, config_cls, config_kwargs): self._test_save_pretrained_selected_adapters(model_id, config_cls, config_kwargs) diff --git a/tests/test_encoder_decoder_models.py b/tests/test_encoder_decoder_models.py index cf200399bf..8e1661d304 100644 --- a/tests/test_encoder_decoder_models.py +++ b/tests/test_encoder_decoder_models.py @@ -67,6 +67,10 @@ def test_prepare_for_training_parametrized(self, test_name, model_id, config_cls def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs): self._test_save_pretrained(model_id, config_cls, config_kwargs) + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) + def test_save_pretrained_regression(self, test_name, model_id, config_cls, config_kwargs): + self._test_save_pretrained_regression(model_id, config_cls, config_kwargs) + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) def test_save_pretrained_selected_adapters(self, test_name, model_id, config_cls, config_kwargs): self._test_save_pretrained_selected_adapters(model_id, config_cls, config_kwargs) diff --git a/tests/test_gpu_examples.py b/tests/test_gpu_examples.py index fbabb8bb6c..ee9725c13e 100644 --- a/tests/test_gpu_examples.py +++ b/tests/test_gpu_examples.py @@ -658,7 +658,7 @@ def setUp(self): from transformers import GPTQConfig self.causal_lm_model_id = "marcsun13/opt-350m-gptq-4bit" - self.quantization_config = GPTQConfig(bits=4, disable_exllama=True) + self.quantization_config = GPTQConfig(bits=4, use_exllama=False) self.tokenizer = AutoTokenizer.from_pretrained(self.causal_lm_model_id) def tearDown(self): diff --git a/tests/testing_common.py b/tests/testing_common.py index 8bb7a104cd..8538d3f482 100644 --- a/tests/testing_common.py +++ b/tests/testing_common.py @@ -269,6 +269,63 @@ def make_inputs_require_grad(module, input, output): self.assertTrue(dummy_output.requires_grad) + def _test_save_pretrained_regression(self, model_id, config_cls, config_kwargs): + # ensure that the weights are randomly initialized + if issubclass(config_cls, LoraConfig): + config_kwargs = config_kwargs.copy() + config_kwargs["init_lora_weights"] = False + if issubclass(config_cls, IA3Config): + config_kwargs = config_kwargs.copy() + config_kwargs["init_ia3_weights"] = False + + model = self.transformers_class.from_pretrained(model_id) + config = config_cls( + base_model_name_or_path=model_id, + **config_kwargs, + ) + model = get_peft_model(model, config) + model = model.to(self.torch_device) + + with tempfile.TemporaryDirectory() as tmp_dirname: + model.save_pretrained(tmp_dirname, safe_serialization=False) + + model_from_pretrained = self.transformers_class.from_pretrained(model_id) + model_from_pretrained = PeftModel.from_pretrained(model_from_pretrained, tmp_dirname) + + # check if the state dicts are equal + if issubclass(config_cls, PromptEncoderConfig): + # For prompt encoding, when loading the whole state_dict, there are differences, therefore, only load + # adapter-specific weights for comparison. + # TODO: is this expected? + state_dict = get_peft_model_state_dict(model, unwrap_compiled=True) + state_dict_from_pretrained = get_peft_model_state_dict(model_from_pretrained, unwrap_compiled=True) + else: + state_dict = get_state_dict(model, unwrap_compiled=True) + state_dict_from_pretrained = get_state_dict(model_from_pretrained, unwrap_compiled=True) + + # check if tensors equal + for key in state_dict.keys(): + self.assertTrue( + torch.allclose( + state_dict[key].to(self.torch_device), state_dict_from_pretrained[key].to(self.torch_device) + ) + ) + + # check if `adapter_model.safetensors` is present + self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin"))) + + # check if `adapter_config.json` is present + self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json"))) + + # check if `model.safetensors` is not present + self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "model.safetensors"))) + + # check if `config.json` is not present + self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json"))) + + self.check_modelcard(tmp_dirname, model) + self.check_config_json(tmp_dirname, model) + def _test_save_pretrained(self, model_id, config_cls, config_kwargs): # ensure that the weights are randomly initialized if issubclass(config_cls, LoraConfig): @@ -311,14 +368,14 @@ def _test_save_pretrained(self, model_id, config_cls, config_kwargs): ) ) - # check if `adapter_model.bin` is present - self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin"))) + # check if `adapter_model.safetensors` is present + self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.safetensors"))) # check if `adapter_config.json` is present self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json"))) - # check if `pytorch_model.bin` is not present - self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "pytorch_model.bin"))) + # check if `model.safetensors` is not present + self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "model.safetensors"))) # check if `config.json` is not present self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json"))) @@ -385,17 +442,17 @@ def _test_save_pretrained_selected_adapters(self, model_id, config_cls, config_k ) ) - # check if `adapter_model.bin` is present - self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin"))) - self.assertTrue(os.path.exists(os.path.join(new_adapter_dir, "adapter_model.bin"))) + # check if `adapter_model.safetensors` is present + self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.safetensors"))) + self.assertTrue(os.path.exists(os.path.join(new_adapter_dir, "adapter_model.safetensors"))) # check if `adapter_config.json` is present self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json"))) self.assertTrue(os.path.exists(os.path.join(new_adapter_dir, "adapter_config.json"))) - # check if `pytorch_model.bin` is not present - self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "pytorch_model.bin"))) - self.assertFalse(os.path.exists(os.path.join(new_adapter_dir, "pytorch_model.bin"))) + # check if `model.safetensors` is not present + self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "model.safetensors"))) + self.assertFalse(os.path.exists(os.path.join(new_adapter_dir, "model.safetensors"))) # check if `config.json` is not present self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json"))) From 449d6570055a1616b4581b833e8c3c226a91dfc9 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 7 Nov 2023 11:04:43 +0000 Subject: [PATCH 3/4] fix final tests' --- tests/test_adaption_prompt.py | 57 ++++++++++++++++++++++++--- tests/test_multitask_prompt_tuning.py | 47 +++++++++++++++++++++- 2 files changed, 97 insertions(+), 7 deletions(-) diff --git a/tests/test_adaption_prompt.py b/tests/test_adaption_prompt.py index d3f2e74140..117c43a427 100644 --- a/tests/test_adaption_prompt.py +++ b/tests/test_adaption_prompt.py @@ -115,6 +115,51 @@ def make_inputs_require_grad(module, input, output): self.assertTrue(dummy_output.requires_grad) + def test_save_pretrained_regression(self) -> None: + seed = 420 + torch.manual_seed(seed) + model = LlamaForCausalLM(self._create_test_llama_config()) + config = AdaptionPromptConfig(adapter_layers=2, adapter_len=4, task_type="CAUSAL_LM") + model = get_peft_model(model, config) + model = model.to(self.torch_device) + + with tempfile.TemporaryDirectory() as tmp_dirname: + model.save_pretrained(tmp_dirname, safe_serialization=False) + + torch.manual_seed(seed) + model_from_pretrained = LlamaForCausalLM(self._create_test_llama_config()) + model_from_pretrained = PeftModel.from_pretrained(model_from_pretrained, tmp_dirname) + + # check if the state dicts are equal + state_dict = get_peft_model_state_dict(model) + state_dict_from_pretrained = get_peft_model_state_dict(model_from_pretrained) + + # check if same keys + self.assertEqual(state_dict.keys(), state_dict_from_pretrained.keys()) + + # Check that the number of saved parameters is 4 -- 2 layers of (tokens and gate). + self.assertEqual(len(list(state_dict.keys())), 4) + + # check if tensors equal + for key in state_dict.keys(): + self.assertTrue( + torch.allclose( + state_dict[key].to(self.torch_device), state_dict_from_pretrained[key].to(self.torch_device) + ) + ) + + # check if `adapter_model.bin` is present + self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin"))) + + # check if `adapter_config.json` is present + self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json"))) + + # check if `model.safetensors` is not present + self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "model.safetensors"))) + + # check if `config.json` is not present + self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json"))) + def test_save_pretrained(self) -> None: seed = 420 torch.manual_seed(seed) @@ -149,13 +194,13 @@ def test_save_pretrained(self) -> None: ) # check if `adapter_model.bin` is present - self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin"))) + self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.safetensors"))) # check if `adapter_config.json` is present self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json"))) - # check if `pytorch_model.bin` is not present - self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "pytorch_model.bin"))) + # check if `model.safetensors` is not present + self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "model.safetensors"))) # check if `config.json` is not present self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json"))) @@ -199,13 +244,13 @@ def test_save_pretrained_selected_adapters(self) -> None: ) # check if `adapter_model.bin` is present - self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin"))) + self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.safetensors"))) # check if `adapter_config.json` is present self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json"))) - # check if `pytorch_model.bin` is not present - self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "pytorch_model.bin"))) + # check if `model.safetensors` is not present + self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "model.safetensors"))) # check if `config.json` is not present self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json"))) diff --git a/tests/test_multitask_prompt_tuning.py b/tests/test_multitask_prompt_tuning.py index 5fc9f36a1d..9aa6b8d7d9 100644 --- a/tests/test_multitask_prompt_tuning.py +++ b/tests/test_multitask_prompt_tuning.py @@ -145,7 +145,52 @@ def test_save_pretrained(self) -> None: ) ) - # check if `adapter_model.bin` is present + # check if `adapter_model.safetensors` is present + self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.safetensors"))) + + # check if `adapter_config.json` is present + self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json"))) + + # check if `pytorch_model.bin` is not present + self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "pytorch_model.bin"))) + + # check if `config.json` is not present + self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json"))) + + def test_save_pretrained_regression(self) -> None: + seed = 420 + torch.manual_seed(seed) + model = LlamaForCausalLM(self._create_test_llama_config()) + model = get_peft_model(model, self._create_multitask_prompt_tuning_config()) + model = model.to(self.torch_device) + + with tempfile.TemporaryDirectory() as tmp_dirname: + model.save_pretrained(tmp_dirname, safe_serialization=False) + + torch.manual_seed(seed) + model_from_pretrained = LlamaForCausalLM(self._create_test_llama_config()) + model_from_pretrained = PeftModel.from_pretrained(model_from_pretrained, tmp_dirname) + + # check if the state dicts are equal + state_dict = get_peft_model_state_dict(model) + + state_dict_from_pretrained = get_peft_model_state_dict(model_from_pretrained) + + # check if same keys + self.assertEqual(state_dict.keys(), state_dict_from_pretrained.keys()) + + # Check that the number of saved parameters is 4 -- 2 layers of (tokens and gate). + self.assertEqual(len(list(state_dict.keys())), 3) + + # check if tensors equal + for key in state_dict.keys(): + self.assertTrue( + torch.allclose( + state_dict[key].to(self.torch_device), state_dict_from_pretrained[key].to(self.torch_device) + ) + ) + + # check if `adapter_model.bin` is present for regression self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin"))) # check if `adapter_config.json` is present From 6c88a13989abb4135bdd98f723821f89c63471d5 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 7 Nov 2023 12:31:09 +0000 Subject: [PATCH 4/4] adapt from suggestion --- tests/test_custom_models.py | 4 +- tests/test_decoder_models.py | 8 ++- tests/test_encoder_decoder_models.py | 8 ++- tests/test_gpu_examples.py | 2 +- tests/testing_common.py | 79 ++++++---------------------- 5 files changed, 31 insertions(+), 70 deletions(-) diff --git a/tests/test_custom_models.py b/tests/test_custom_models.py index 13a98b6d32..4998751d06 100644 --- a/tests/test_custom_models.py +++ b/tests/test_custom_models.py @@ -409,8 +409,8 @@ def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs): self._test_save_pretrained(model_id, config_cls, config_kwargs) @parameterized.expand(TEST_CASES) - def test_save_pretrained_regression(self, test_name, model_id, config_cls, config_kwargs): - self._test_save_pretrained_regression(model_id, config_cls, config_kwargs) + def test_save_pretrained_pickle(self, test_name, model_id, config_cls, config_kwargs): + self._test_save_pretrained(model_id, config_cls, config_kwargs, safe_serialization=False) @parameterized.expand(TEST_CASES) def test_from_pretrained_config_construction(self, test_name, model_id, config_cls, config_kwargs): diff --git a/tests/test_decoder_models.py b/tests/test_decoder_models.py index 622d978e60..9043cc2475 100644 --- a/tests/test_decoder_models.py +++ b/tests/test_decoder_models.py @@ -81,13 +81,17 @@ def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs): self._test_save_pretrained(model_id, config_cls, config_kwargs) @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) - def test_save_pretrained_regression(self, test_name, model_id, config_cls, config_kwargs): - self._test_save_pretrained_regression(model_id, config_cls, config_kwargs) + def test_save_pretrained_pickle(self, test_name, model_id, config_cls, config_kwargs): + self._test_save_pretrained(model_id, config_cls, config_kwargs, safe_serialization=False) @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) def test_save_pretrained_selected_adapters(self, test_name, model_id, config_cls, config_kwargs): self._test_save_pretrained_selected_adapters(model_id, config_cls, config_kwargs) + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) + def test_save_pretrained_selected_adapters_pickle(self, test_name, model_id, config_cls, config_kwargs): + self._test_save_pretrained_selected_adapters(model_id, config_cls, config_kwargs, safe_serialization=False) + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) def test_from_pretrained_config_construction(self, test_name, model_id, config_cls, config_kwargs): self._test_from_pretrained_config_construction(model_id, config_cls, config_kwargs) diff --git a/tests/test_encoder_decoder_models.py b/tests/test_encoder_decoder_models.py index 8e1661d304..17e2a065de 100644 --- a/tests/test_encoder_decoder_models.py +++ b/tests/test_encoder_decoder_models.py @@ -68,13 +68,17 @@ def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs): self._test_save_pretrained(model_id, config_cls, config_kwargs) @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) - def test_save_pretrained_regression(self, test_name, model_id, config_cls, config_kwargs): - self._test_save_pretrained_regression(model_id, config_cls, config_kwargs) + def test_save_pretrained_pickle(self, test_name, model_id, config_cls, config_kwargs): + self._test_save_pretrained(model_id, config_cls, config_kwargs, safe_serialization=False) @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) def test_save_pretrained_selected_adapters(self, test_name, model_id, config_cls, config_kwargs): self._test_save_pretrained_selected_adapters(model_id, config_cls, config_kwargs) + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) + def test_save_pretrained_selected_adapters_pickle(self, test_name, model_id, config_cls, config_kwargs): + self._test_save_pretrained_selected_adapters(model_id, config_cls, config_kwargs, safe_serialization=False) + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) def test_from_pretrained_config_construction(self, test_name, model_id, config_cls, config_kwargs): self._test_from_pretrained_config_construction(model_id, config_cls, config_kwargs) diff --git a/tests/test_gpu_examples.py b/tests/test_gpu_examples.py index ee9725c13e..fbabb8bb6c 100644 --- a/tests/test_gpu_examples.py +++ b/tests/test_gpu_examples.py @@ -658,7 +658,7 @@ def setUp(self): from transformers import GPTQConfig self.causal_lm_model_id = "marcsun13/opt-350m-gptq-4bit" - self.quantization_config = GPTQConfig(bits=4, use_exllama=False) + self.quantization_config = GPTQConfig(bits=4, disable_exllama=True) self.tokenizer = AutoTokenizer.from_pretrained(self.causal_lm_model_id) def tearDown(self): diff --git a/tests/testing_common.py b/tests/testing_common.py index 8538d3f482..e47944a434 100644 --- a/tests/testing_common.py +++ b/tests/testing_common.py @@ -269,7 +269,7 @@ def make_inputs_require_grad(module, input, output): self.assertTrue(dummy_output.requires_grad) - def _test_save_pretrained_regression(self, model_id, config_cls, config_kwargs): + def _test_save_pretrained(self, model_id, config_cls, config_kwargs, safe_serialization=True): # ensure that the weights are randomly initialized if issubclass(config_cls, LoraConfig): config_kwargs = config_kwargs.copy() @@ -287,64 +287,10 @@ def _test_save_pretrained_regression(self, model_id, config_cls, config_kwargs): model = model.to(self.torch_device) with tempfile.TemporaryDirectory() as tmp_dirname: - model.save_pretrained(tmp_dirname, safe_serialization=False) - - model_from_pretrained = self.transformers_class.from_pretrained(model_id) - model_from_pretrained = PeftModel.from_pretrained(model_from_pretrained, tmp_dirname) - - # check if the state dicts are equal - if issubclass(config_cls, PromptEncoderConfig): - # For prompt encoding, when loading the whole state_dict, there are differences, therefore, only load - # adapter-specific weights for comparison. - # TODO: is this expected? - state_dict = get_peft_model_state_dict(model, unwrap_compiled=True) - state_dict_from_pretrained = get_peft_model_state_dict(model_from_pretrained, unwrap_compiled=True) + if safe_serialization: + model.save_pretrained(tmp_dirname) else: - state_dict = get_state_dict(model, unwrap_compiled=True) - state_dict_from_pretrained = get_state_dict(model_from_pretrained, unwrap_compiled=True) - - # check if tensors equal - for key in state_dict.keys(): - self.assertTrue( - torch.allclose( - state_dict[key].to(self.torch_device), state_dict_from_pretrained[key].to(self.torch_device) - ) - ) - - # check if `adapter_model.safetensors` is present - self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin"))) - - # check if `adapter_config.json` is present - self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json"))) - - # check if `model.safetensors` is not present - self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "model.safetensors"))) - - # check if `config.json` is not present - self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json"))) - - self.check_modelcard(tmp_dirname, model) - self.check_config_json(tmp_dirname, model) - - def _test_save_pretrained(self, model_id, config_cls, config_kwargs): - # ensure that the weights are randomly initialized - if issubclass(config_cls, LoraConfig): - config_kwargs = config_kwargs.copy() - config_kwargs["init_lora_weights"] = False - if issubclass(config_cls, IA3Config): - config_kwargs = config_kwargs.copy() - config_kwargs["init_ia3_weights"] = False - - model = self.transformers_class.from_pretrained(model_id) - config = config_cls( - base_model_name_or_path=model_id, - **config_kwargs, - ) - model = get_peft_model(model, config) - model = model.to(self.torch_device) - - with tempfile.TemporaryDirectory() as tmp_dirname: - model.save_pretrained(tmp_dirname) + model.save_pretrained(tmp_dirname, safe_serialization=False) model_from_pretrained = self.transformers_class.from_pretrained(model_id) model_from_pretrained = PeftModel.from_pretrained(model_from_pretrained, tmp_dirname) @@ -368,8 +314,10 @@ def _test_save_pretrained(self, model_id, config_cls, config_kwargs): ) ) + target_adapter_filename = "adapter_model.safetensors" if safe_serialization else "adapter_model.bin" + # check if `adapter_model.safetensors` is present - self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.safetensors"))) + self.assertTrue(os.path.exists(os.path.join(tmp_dirname, target_adapter_filename))) # check if `adapter_config.json` is present self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json"))) @@ -383,7 +331,7 @@ def _test_save_pretrained(self, model_id, config_cls, config_kwargs): self.check_modelcard(tmp_dirname, model) self.check_config_json(tmp_dirname, model) - def _test_save_pretrained_selected_adapters(self, model_id, config_cls, config_kwargs): + def _test_save_pretrained_selected_adapters(self, model_id, config_cls, config_kwargs, safe_serialization=True): if issubclass(config_cls, AdaLoraConfig): # AdaLora does not support adding more than 1 adapter return @@ -412,7 +360,10 @@ def _test_save_pretrained_selected_adapters(self, model_id, config_cls, config_k model.add_adapter("new_adapter", new_adapter_config) with tempfile.TemporaryDirectory() as tmp_dirname: - model.save_pretrained(tmp_dirname) + if safe_serialization: + model.save_pretrained(tmp_dirname) + else: + model.save_pretrained(tmp_dirname, safe_serialization=False) model_from_pretrained = self.transformers_class.from_pretrained(model_id) model_from_pretrained = PeftModel.from_pretrained(model_from_pretrained, tmp_dirname) @@ -442,9 +393,11 @@ def _test_save_pretrained_selected_adapters(self, model_id, config_cls, config_k ) ) + target_adapter_filename = "adapter_model.safetensors" if safe_serialization else "adapter_model.bin" + # check if `adapter_model.safetensors` is present - self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.safetensors"))) - self.assertTrue(os.path.exists(os.path.join(new_adapter_dir, "adapter_model.safetensors"))) + self.assertTrue(os.path.exists(os.path.join(tmp_dirname, target_adapter_filename))) + self.assertTrue(os.path.exists(os.path.join(new_adapter_dir, target_adapter_filename))) # check if `adapter_config.json` is present self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json")))