Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

FEAT: Make safe serialization the default one #1088

Merged
merged 5 commits into from
Nov 15, 2023
Merged

FEAT: Make safe serialization the default one #1088

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8e924fb
make safe serialization the default one
younesbelkada Nov 7, 2023
43c8f69
adapt tests
younesbelkada Nov 7, 2023
449d657
fix final tests'
younesbelkada Nov 7, 2023
6c88a13
adapt from suggestion
younesbelkada Nov 7, 2023
06ace9b
Merge remote-tracking branch 'upstream/main' into make-st-default
younesbelkada Nov 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/peft/peft_model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ def peft_config(self, value: Dict[str, PeftConfig]):
def save_pretrained(
self,
save_directory: str,
safe_serialization: bool = False,
safe_serialization: bool = True,
selected_adapters: Optional[List[str]] = None,
**kwargs: Any,
):
Expand All @@ -168,6 +168,8 @@ def save_pretrained(
save_directory (`str`):
Directory where the adapter model and configuration files will be saved (will be created if it does not
exist).
safe_serialization (`bool`, *optional*):
Whether to save the adapter files in safetensors format.
kwargs (additional keyword arguments, *optional*):
Additional keyword arguments passed along to the `push_to_hub` method.
"""
Expand Down
4 changes: 4 additions & 0 deletions tests/test_custom_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,10 @@ def test_prepare_for_training_parametrized(self, test_name, model_id, config_cls
def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs):
self._test_save_pretrained(model_id, config_cls, config_kwargs)

@parameterized.expand(TEST_CASES)
def test_save_pretrained_regression(self, test_name, model_id, config_cls, config_kwargs):
self._test_save_pretrained_regression(model_id, config_cls, config_kwargs)

@parameterized.expand(TEST_CASES)
def test_from_pretrained_config_construction(self, test_name, model_id, config_cls, config_kwargs):
self._test_from_pretrained_config_construction(model_id, config_cls, config_kwargs)
Expand Down
4 changes: 4 additions & 0 deletions tests/test_decoder_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,10 @@ def test_prepare_for_training_parametrized(self, test_name, model_id, config_cls
def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs):
self._test_save_pretrained(model_id, config_cls, config_kwargs)

@parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
def test_save_pretrained_regression(self, test_name, model_id, config_cls, config_kwargs):
self._test_save_pretrained_regression(model_id, config_cls, config_kwargs)

@parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
def test_save_pretrained_selected_adapters(self, test_name, model_id, config_cls, config_kwargs):
self._test_save_pretrained_selected_adapters(model_id, config_cls, config_kwargs)
Expand Down
4 changes: 4 additions & 0 deletions tests/test_encoder_decoder_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,10 @@ def test_prepare_for_training_parametrized(self, test_name, model_id, config_cls
def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs):
self._test_save_pretrained(model_id, config_cls, config_kwargs)

@parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
def test_save_pretrained_regression(self, test_name, model_id, config_cls, config_kwargs):
self._test_save_pretrained_regression(model_id, config_cls, config_kwargs)

@parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
def test_save_pretrained_selected_adapters(self, test_name, model_id, config_cls, config_kwargs):
self._test_save_pretrained_selected_adapters(model_id, config_cls, config_kwargs)
Expand Down
2 changes: 1 addition & 1 deletion tests/test_gpu_examples.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -658,7 +658,7 @@ def setUp(self):
from transformers import GPTQConfig

self.causal_lm_model_id = "marcsun13/opt-350m-gptq-4bit"
self.quantization_config = GPTQConfig(bits=4, disable_exllama=True)
self.quantization_config = GPTQConfig(bits=4, use_exllama=False)
self.tokenizer = AutoTokenizer.from_pretrained(self.causal_lm_model_id)

def tearDown(self):
Expand Down
77 changes: 67 additions & 10 deletions tests/testing_common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,63 @@ def make_inputs_require_grad(module, input, output):

self.assertTrue(dummy_output.requires_grad)

def _test_save_pretrained_regression(self, model_id, config_cls, config_kwargs):
# ensure that the weights are randomly initialized
if issubclass(config_cls, LoraConfig):
config_kwargs = config_kwargs.copy()
config_kwargs["init_lora_weights"] = False
if issubclass(config_cls, IA3Config):
config_kwargs = config_kwargs.copy()
config_kwargs["init_ia3_weights"] = False

model = self.transformers_class.from_pretrained(model_id)
config = config_cls(
base_model_name_or_path=model_id,
**config_kwargs,
)
model = get_peft_model(model, config)
model = model.to(self.torch_device)

with tempfile.TemporaryDirectory() as tmp_dirname:
model.save_pretrained(tmp_dirname, safe_serialization=False)

model_from_pretrained = self.transformers_class.from_pretrained(model_id)
model_from_pretrained = PeftModel.from_pretrained(model_from_pretrained, tmp_dirname)

# check if the state dicts are equal
if issubclass(config_cls, PromptEncoderConfig):
# For prompt encoding, when loading the whole state_dict, there are differences, therefore, only load
# adapter-specific weights for comparison.
# TODO: is this expected?
state_dict = get_peft_model_state_dict(model, unwrap_compiled=True)
state_dict_from_pretrained = get_peft_model_state_dict(model_from_pretrained, unwrap_compiled=True)
else:
state_dict = get_state_dict(model, unwrap_compiled=True)
state_dict_from_pretrained = get_state_dict(model_from_pretrained, unwrap_compiled=True)

# check if tensors equal
for key in state_dict.keys():
self.assertTrue(
torch.allclose(
state_dict[key].to(self.torch_device), state_dict_from_pretrained[key].to(self.torch_device)
)
)

# check if `adapter_model.safetensors` is present
self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin")))

# check if `adapter_config.json` is present
self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json")))

# check if `model.safetensors` is not present
self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "model.safetensors")))

# check if `config.json` is not present
self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json")))

self.check_modelcard(tmp_dirname, model)
self.check_config_json(tmp_dirname, model)

def _test_save_pretrained(self, model_id, config_cls, config_kwargs):
# ensure that the weights are randomly initialized
if issubclass(config_cls, LoraConfig):
Expand Down Expand Up @@ -311,14 +368,14 @@ def _test_save_pretrained(self, model_id, config_cls, config_kwargs):
)
)

# check if `adapter_model.bin` is present
self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin")))
# check if `adapter_model.safetensors` is present
self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.safetensors")))

# check if `adapter_config.json` is present
self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json")))

# check if `pytorch_model.bin` is not present
self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "pytorch_model.bin")))
# check if `model.safetensors` is not present
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This change in necessary as now in transformers we don't save anymore pytorch-model.bin but the safetensors model

self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "model.safetensors")))

# check if `config.json` is not present
self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json")))
Expand Down Expand Up @@ -385,17 +442,17 @@ def _test_save_pretrained_selected_adapters(self, model_id, config_cls, config_k
)
)

# check if `adapter_model.bin` is present
self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin")))
self.assertTrue(os.path.exists(os.path.join(new_adapter_dir, "adapter_model.bin")))
# check if `adapter_model.safetensors` is present
self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.safetensors")))
self.assertTrue(os.path.exists(os.path.join(new_adapter_dir, "adapter_model.safetensors")))

# check if `adapter_config.json` is present
self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json")))
self.assertTrue(os.path.exists(os.path.join(new_adapter_dir, "adapter_config.json")))

# check if `pytorch_model.bin` is not present
self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "pytorch_model.bin")))
self.assertFalse(os.path.exists(os.path.join(new_adapter_dir, "pytorch_model.bin")))
# check if `model.safetensors` is not present
self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "model.safetensors")))
self.assertFalse(os.path.exists(os.path.join(new_adapter_dir, "model.safetensors")))

# check if `config.json` is not present
self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json")))
Expand Down