From 61d8277fd5ed242073e70932775e72dad84fd254 Mon Sep 17 00:00:00 2001 From: not-lain Date: Wed, 21 Feb 2024 14:35:39 +0100 Subject: [PATCH 01/14] add `push_to_hub` to pipeline --- src/transformers/pipelines/base.py | 45 ++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index 758484107b76f2..e2d3a0f8241c7d 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -36,7 +36,9 @@ from ..tokenization_utils import PreTrainedTokenizer from ..utils import ( ModelOutput, + PushToHubMixin, add_end_docstrings, + copy_func, infer_framework, is_tf_available, is_torch_available, @@ -779,7 +781,7 @@ def build_pipeline_init_args( @add_end_docstrings(build_pipeline_init_args(has_tokenizer=True, has_feature_extractor=True, has_image_processor=True)) -class Pipeline(_ScikitCompat): +class Pipeline(_ScikitCompat, PushToHubMixin): """ The Pipeline class is the class from which all pipelines inherit. Refer to this class for methods shared across different pipelines. @@ -892,7 +894,12 @@ def __init__( # then we should keep working self.image_processor = self.feature_extractor - def save_pretrained(self, save_directory: str, safe_serialization: bool = True): + def save_pretrained( + self, + save_directory: str, + safe_serialization: bool = True, + **kwargs, + ): """ Save the pipeline's model and tokenizer. @@ -902,6 +909,8 @@ def save_pretrained(self, save_directory: str, safe_serialization: bool = True): safe_serialization (`str`): Whether to save the model using `safetensors` or the traditional way for PyTorch or Tensorflow. """ + self._set_token_in_kwargs(kwargs) + if os.path.isfile(save_directory): logger.error(f"Provided path ({save_directory}) should be a directory, not a file") return @@ -942,6 +951,31 @@ def save_pretrained(self, save_directory: str, safe_serialization: bool = True): if self.modelcard is not None: self.modelcard.save_pretrained(save_directory) + @staticmethod + def _set_token_in_kwargs(kwargs, token=None): + """Temporary method to deal with `token` and `use_auth_token`. + This method is to avoid apply the same changes in all model config classes that overwrite `from_pretrained`. + Need to clean up `use_auth_token` in a follow PR. + """ + # Some model config classes like CLIP define their own `from_pretrained` without the new argument `token` yet. + if token is None: + token = kwargs.pop("token", None) + use_auth_token = kwargs.pop("use_auth_token", None) + + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if token is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + token = use_auth_token + + if token is not None: + kwargs["token"] = token + def transform(self, X): """ Scikit / Keras interface to transformers' pipelines. This method will forward to __call__(). @@ -1211,6 +1245,13 @@ def iterate(self, inputs, preprocess_params, forward_params, postprocess_params) yield self.run_single(input_, preprocess_params, forward_params, postprocess_params) +Pipeline.push_to_hub = copy_func(Pipeline.push_to_hub) +if Pipeline.push_to_hub.__doc__ is not None: + Pipeline.push_to_hub.__doc__ = Pipeline.push_to_hub.__doc__.format( + object="pipe", object_class="pipeline", object_files="pipeline file" + ) + + class ChunkPipeline(Pipeline): def run_single(self, inputs, preprocess_params, forward_params, postprocess_params): all_outputs = [] From 910581d113fec053f2bedaa537123198bb3f593d Mon Sep 17 00:00:00 2001 From: not-lain Date: Thu, 22 Feb 2024 11:51:26 +0100 Subject: [PATCH 02/14] fix docs --- src/transformers/pipelines/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index e2d3a0f8241c7d..b8cd283e04aaa3 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -1249,7 +1249,7 @@ def iterate(self, inputs, preprocess_params, forward_params, postprocess_params) if Pipeline.push_to_hub.__doc__ is not None: Pipeline.push_to_hub.__doc__ = Pipeline.push_to_hub.__doc__.format( object="pipe", object_class="pipeline", object_files="pipeline file" - ) + ).replace(".from_pretrained","") class ChunkPipeline(Pipeline): From 4be0c3510f834f696dc55ffd0ede1d340e7bf719 Mon Sep 17 00:00:00 2001 From: not-lain Date: Thu, 22 Feb 2024 11:57:16 +0100 Subject: [PATCH 03/14] format with ruff --- src/transformers/pipelines/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index b8cd283e04aaa3..37b19ce0a76afb 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -1249,7 +1249,7 @@ def iterate(self, inputs, preprocess_params, forward_params, postprocess_params) if Pipeline.push_to_hub.__doc__ is not None: Pipeline.push_to_hub.__doc__ = Pipeline.push_to_hub.__doc__.format( object="pipe", object_class="pipeline", object_files="pipeline file" - ).replace(".from_pretrained","") + ).replace(".from_pretrained", "") class ChunkPipeline(Pipeline): From 8fc4ac54653158cbb09211473272a1c7b2b1ce89 Mon Sep 17 00:00:00 2001 From: not-lain Date: Tue, 27 Feb 2024 15:36:47 +0100 Subject: [PATCH 04/14] update save_pretrained --- src/transformers/pipelines/base.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index 37b19ce0a76afb..531c40b2f28891 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -896,7 +896,7 @@ def __init__( def save_pretrained( self, - save_directory: str, + save_directory: Union[str, os.PathLike], safe_serialization: bool = True, **kwargs, ): @@ -904,7 +904,7 @@ def save_pretrained( Save the pipeline's model and tokenizer. Args: - save_directory (`str`): + save_directory (`str` or `os.PathLike`): A path to the directory where to saved. It will be created if it doesn't exist. safe_serialization (`str`): Whether to save the model using `safetensors` or the traditional way for PyTorch or Tensorflow. @@ -937,16 +937,17 @@ def save_pretrained( # Save the pipeline custom code custom_object_save(self, save_directory) - self.model.save_pretrained(save_directory, safe_serialization=safe_serialization) + kwargs["safe_serialization"] = safe_serialization + self.model.save_pretrained(save_directory, **kwargs) if self.tokenizer is not None: - self.tokenizer.save_pretrained(save_directory) + self.tokenizer.save_pretrained(save_directory,**kwargs) if self.feature_extractor is not None: - self.feature_extractor.save_pretrained(save_directory) + self.feature_extractor.save_pretrained(save_directory,**kwargs) if self.image_processor is not None: - self.image_processor.save_pretrained(save_directory) + self.image_processor.save_pretrained(save_directory,**kwargs) if self.modelcard is not None: self.modelcard.save_pretrained(save_directory) From 25bdabedb5975e45501cfec737bff67f3a113015 Mon Sep 17 00:00:00 2001 From: not-lain Date: Wed, 28 Feb 2024 17:15:19 +0100 Subject: [PATCH 05/14] update save_pretrained --- src/transformers/pipelines/base.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index 531c40b2f28891..0b12e281d73e25 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -898,6 +898,7 @@ def save_pretrained( self, save_directory: Union[str, os.PathLike], safe_serialization: bool = True, + token: Optional[Union[str, bool]] = None, **kwargs, ): """ @@ -908,8 +909,13 @@ def save_pretrained( A path to the directory where to saved. It will be created if it doesn't exist. safe_serialization (`str`): Whether to save the model using `safetensors` or the traditional way for PyTorch or Tensorflow. + token (`str` or `bool`, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use + the token generated when running `huggingface-cli login` (stored in `~/.huggingface`). + kwargs (`Dict[str, Any]`, *optional*): + Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. """ - self._set_token_in_kwargs(kwargs) + self._set_token_in_kwargs(kwargs, token=token) if os.path.isfile(save_directory): logger.error(f"Provided path ({save_directory}) should be a directory, not a file") @@ -937,17 +943,20 @@ def save_pretrained( # Save the pipeline custom code custom_object_save(self, save_directory) + # TODO: + # depricate the safe_serialization parameter and use kwargs instead + # or update the save_pretrained to get all the parameters such as max_shard_size, ... kwargs["safe_serialization"] = safe_serialization self.model.save_pretrained(save_directory, **kwargs) if self.tokenizer is not None: - self.tokenizer.save_pretrained(save_directory,**kwargs) + self.tokenizer.save_pretrained(save_directory, **kwargs) if self.feature_extractor is not None: - self.feature_extractor.save_pretrained(save_directory,**kwargs) + self.feature_extractor.save_pretrained(save_directory, **kwargs) if self.image_processor is not None: - self.image_processor.save_pretrained(save_directory,**kwargs) + self.image_processor.save_pretrained(save_directory, **kwargs) if self.modelcard is not None: self.modelcard.save_pretrained(save_directory) From 547edb60b217578393a41c3c1ce8b65955f9a1cd Mon Sep 17 00:00:00 2001 From: not-lain Date: Fri, 1 Mar 2024 15:38:07 +0100 Subject: [PATCH 06/14] remove unnecessary comment --- src/transformers/pipelines/base.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index 0b12e281d73e25..d672d65fa8c037 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -943,9 +943,6 @@ def save_pretrained( # Save the pipeline custom code custom_object_save(self, save_directory) - # TODO: - # depricate the safe_serialization parameter and use kwargs instead - # or update the save_pretrained to get all the parameters such as max_shard_size, ... kwargs["safe_serialization"] = safe_serialization self.model.save_pretrained(save_directory, **kwargs) From e69e24170b9c25792359f9dec8556e4fe6300846 Mon Sep 17 00:00:00 2001 From: not-lain Date: Wed, 6 Mar 2024 21:56:26 +0100 Subject: [PATCH 07/14] switch to push_to_hub method in DynamicPipelineTester --- tests/pipelines/test_pipelines_common.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/pipelines/test_pipelines_common.py b/tests/pipelines/test_pipelines_common.py index 5e3e15f39c10ea..82af40fbc0b712 100644 --- a/tests/pipelines/test_pipelines_common.py +++ b/tests/pipelines/test_pipelines_common.py @@ -823,9 +823,6 @@ def test_push_to_hub_dynamic_pipeline(self): model = BertForSequenceClassification(config).eval() with tempfile.TemporaryDirectory() as tmp_dir: - create_repo(f"{USER}/test-dynamic-pipeline", token=self._token) - repo = Repository(tmp_dir, clone_from=f"{USER}/test-dynamic-pipeline", token=self._token) - vocab_file = os.path.join(tmp_dir, "vocab.txt") with open(vocab_file, "w", encoding="utf-8") as vocab_writer: vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens])) @@ -837,7 +834,7 @@ def test_push_to_hub_dynamic_pipeline(self): del PIPELINE_REGISTRY.supported_tasks["pair-classification"] classifier.save_pretrained(tmp_dir) - # checks + # checks if the configuration has been added after calling the save_pretrained method self.assertDictEqual( classifier.model.config.custom_pipelines, { @@ -848,8 +845,8 @@ def test_push_to_hub_dynamic_pipeline(self): } }, ) - - repo.push_to_hub() + # use push_to_hub method to push the pipeline + classifier.push_to_hub(f"{USER}/test-dynamic-pipeline", token=self._token) # Fails if the user forget to pass along `trust_remote_code=True` with self.assertRaises(ValueError): From 6b108db8d09cc3a80bae9e5df2e5af1dc4a9c417 Mon Sep 17 00:00:00 2001 From: not-lain Date: Wed, 6 Mar 2024 22:55:45 +0100 Subject: [PATCH 08/14] remove unused imports --- tests/pipelines/test_pipelines_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipelines/test_pipelines_common.py b/tests/pipelines/test_pipelines_common.py index 82af40fbc0b712..0c936c3850270f 100644 --- a/tests/pipelines/test_pipelines_common.py +++ b/tests/pipelines/test_pipelines_common.py @@ -22,7 +22,7 @@ import datasets import numpy as np -from huggingface_hub import HfFolder, Repository, create_repo, delete_repo +from huggingface_hub import HfFolder, delete_repo from requests.exceptions import HTTPError from transformers import ( From e777027e967a435cf273b07f05056d554d8db2b4 Mon Sep 17 00:00:00 2001 From: Lain Date: Fri, 29 Mar 2024 04:17:16 +0100 Subject: [PATCH 09/14] update docs for add_new_pipeline --- docs/source/de/add_new_pipeline.md | 8 ++------ docs/source/en/add_new_pipeline.md | 8 ++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/docs/source/de/add_new_pipeline.md b/docs/source/de/add_new_pipeline.md index f5e64be7db310f..47d93e90ac1494 100644 --- a/docs/source/de/add_new_pipeline.md +++ b/docs/source/de/add_new_pipeline.md @@ -208,14 +208,10 @@ from transformers import pipeline classifier = pipeline("pair-classification", model="sgugger/finetuned-bert-mrpc") ``` -Dann können wir sie auf dem Hub mit der Methode `save_pretrained` in einem `Repository` freigeben: +Dann können wir sie auf dem Hub mit der Methode `push_to_hub` freigeben: ```py -from huggingface_hub import Repository - -repo = Repository("test-dynamic-pipeline", clone_from="{your_username}/test-dynamic-pipeline") -classifier.save_pretrained("test-dynamic-pipeline") -repo.push_to_hub() +classifier.push_to_hub("test-dynamic-pipeline") ``` Dadurch wird die Datei, in der Sie `PairClassificationPipeline` definiert haben, in den Ordner `"test-dynamic-pipeline"` kopiert, diff --git a/docs/source/en/add_new_pipeline.md b/docs/source/en/add_new_pipeline.md index 9e10c310f07f39..1e5b95e9b48cfc 100644 --- a/docs/source/en/add_new_pipeline.md +++ b/docs/source/en/add_new_pipeline.md @@ -208,14 +208,10 @@ from transformers import pipeline classifier = pipeline("pair-classification", model="sgugger/finetuned-bert-mrpc") ``` -Then we can share it on the Hub by using the `save_pretrained` method in a `Repository`: +Then we can share it on the Hub by using the `push_to_hub` method: ```py -from huggingface_hub import Repository - -repo = Repository("test-dynamic-pipeline", clone_from="{your_username}/test-dynamic-pipeline") -classifier.save_pretrained("test-dynamic-pipeline") -repo.push_to_hub() +classifier.push_to_hub("test-dynamic-pipeline") ``` This will copy the file where you defined `PairClassificationPipeline` inside the folder `"test-dynamic-pipeline"`, From b465ac52068a0bcfc1dff8ce8ccf6b7c9362f089 Mon Sep 17 00:00:00 2001 From: Lain Date: Fri, 29 Mar 2024 04:27:57 +0100 Subject: [PATCH 10/14] fix docs for add_new_pipeline --- docs/source/es/add_new_pipeline.md | 8 ++------ docs/source/it/add_new_pipeline.md | 8 ++------ docs/source/ko/add_new_pipeline.md | 8 ++------ 3 files changed, 6 insertions(+), 18 deletions(-) diff --git a/docs/source/es/add_new_pipeline.md b/docs/source/es/add_new_pipeline.md index 289444350dfa35..4ccacbd18a1853 100644 --- a/docs/source/es/add_new_pipeline.md +++ b/docs/source/es/add_new_pipeline.md @@ -212,14 +212,10 @@ from transformers import pipeline classifier = pipeline("pair-classification", model="sgugger/finetuned-bert-mrpc") ``` -Ahora podemos compartirlo en el Hub usando el método `save_pretrained` (guardar pre-entrenado) en un `Repository`: +Ahora podemos compartirlo en el Hub usando el método `save_pretrained`: ```py -from huggingface_hub import Repository - -repo = Repository("test-dynamic-pipeline", clone_from="{your_username}/test-dynamic-pipeline") -classifier.save_pretrained("test-dynamic-pipeline") -repo.push_to_hub() +classifier.push_to_hub("test-dynamic-pipeline") ``` Esto copiará el archivo donde definiste `PairClassificationPipeline` dentro de la carpeta `"test-dynamic-pipeline"`, diff --git a/docs/source/it/add_new_pipeline.md b/docs/source/it/add_new_pipeline.md index cd42e5cc2cd3d9..b62140e1741b1c 100644 --- a/docs/source/it/add_new_pipeline.md +++ b/docs/source/it/add_new_pipeline.md @@ -202,14 +202,10 @@ from transformers import pipeline classifier = pipeline("pair-classification", model="sgugger/finetuned-bert-mrpc") ``` -Successivamente possiamo condividerlo sull'Hub usando il metodo `save_pretrained` in un `Repository`: +Successivamente possiamo condividerlo sull'Hub usando il metodo `push_to_hub` in un `Repository`: ```py -from huggingface_hub import Repository - -repo = Repository("test-dynamic-pipeline", clone_from="{your_username}/test-dynamic-pipeline") -classifier.save_pretrained("test-dynamic-pipeline") -repo.push_to_hub() +classifier.push_to_hub("test-dynamic-pipeline") ``` Questo codice copierà il file dove è stato definitp `PairClassificationPipeline` all'interno della cartella `"test-dynamic-pipeline"`, diff --git a/docs/source/ko/add_new_pipeline.md b/docs/source/ko/add_new_pipeline.md index 9ddd4981154a37..42c9b57c9d7be6 100644 --- a/docs/source/ko/add_new_pipeline.md +++ b/docs/source/ko/add_new_pipeline.md @@ -203,14 +203,10 @@ from transformers import pipeline classifier = pipeline("pair-classification", model="sgugger/finetuned-bert-mrpc") ``` -그런 다음 `Repository`의 `save_pretrained` 메소드를 사용하여 허브에 공유할 수 있습니다: +그런 다음 `push_to_hub` 메소드를 사용하여 허브에 공유할 수 있습니다: ```py -from huggingface_hub import Repository - -repo = Repository("test-dynamic-pipeline", clone_from="{your_username}/test-dynamic-pipeline") -classifier.save_pretrained("test-dynamic-pipeline") -repo.push_to_hub() +classifier.push_to_hub("test-dynamic-pipeline") ``` 이렇게 하면 "test-dynamic-pipeline" 폴더 내에 `PairClassificationPipeline`을 정의한 파일이 복사되며, 파이프라인의 모델과 토크나이저도 저장한 후, `{your_username}/test-dynamic-pipeline` 저장소에 있는 모든 것을 푸시합니다. From 8b524fc4a461f369b01ed2752ee7de2f0576af98 Mon Sep 17 00:00:00 2001 From: Lain Date: Thu, 4 Apr 2024 15:51:52 +0100 Subject: [PATCH 11/14] add comment --- src/transformers/pipelines/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index d672d65fa8c037..730d0e0901a7a8 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -958,6 +958,7 @@ def save_pretrained( if self.modelcard is not None: self.modelcard.save_pretrained(save_directory) + # Copied from src/transformers/configuration_utils.py @staticmethod def _set_token_in_kwargs(kwargs, token=None): """Temporary method to deal with `token` and `use_auth_token`. From 4872b570441b3294dd6c9680678e195a2fe17787 Mon Sep 17 00:00:00 2001 From: Lain Date: Fri, 5 Apr 2024 13:06:07 +0100 Subject: [PATCH 12/14] fix italien docs --- docs/source/it/add_new_pipeline.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/it/add_new_pipeline.md b/docs/source/it/add_new_pipeline.md index b62140e1741b1c..fcc4da1899a2b1 100644 --- a/docs/source/it/add_new_pipeline.md +++ b/docs/source/it/add_new_pipeline.md @@ -202,7 +202,7 @@ from transformers import pipeline classifier = pipeline("pair-classification", model="sgugger/finetuned-bert-mrpc") ``` -Successivamente possiamo condividerlo sull'Hub usando il metodo `push_to_hub` in un `Repository`: +Successivamente possiamo condividerlo sull'Hub usando il metodo `push_to_hub` ```py classifier.push_to_hub("test-dynamic-pipeline") From 5912999886e398eec5e78966ca4385d86c267335 Mon Sep 17 00:00:00 2001 From: Lain Date: Mon, 15 Apr 2024 15:31:55 +0100 Subject: [PATCH 13/14] changes to token retrieval for pipelines --- src/transformers/pipelines/base.py | 42 +++++++++--------------------- 1 file changed, 12 insertions(+), 30 deletions(-) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index 92146d0bb8a22f..182af4011bb7f3 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -922,13 +922,21 @@ def save_pretrained( A path to the directory where to saved. It will be created if it doesn't exist. safe_serialization (`str`): Whether to save the model using `safetensors` or the traditional way for PyTorch or Tensorflow. - token (`str` or `bool`, *optional*): - The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use - the token generated when running `huggingface-cli login` (stored in `~/.huggingface`). kwargs (`Dict[str, Any]`, *optional*): Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. """ - self._set_token_in_kwargs(kwargs, token=token) + use_auth_token = kwargs.pop("use_auth_token", None) + + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if kwargs.get("token", None) is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + kwargs["token"] = use_auth_token if os.path.isfile(save_directory): logger.error(f"Provided path ({save_directory}) should be a directory, not a file") @@ -971,32 +979,6 @@ def save_pretrained( if self.modelcard is not None: self.modelcard.save_pretrained(save_directory) - # Copied from src/transformers/configuration_utils.py - @staticmethod - def _set_token_in_kwargs(kwargs, token=None): - """Temporary method to deal with `token` and `use_auth_token`. - This method is to avoid apply the same changes in all model config classes that overwrite `from_pretrained`. - Need to clean up `use_auth_token` in a follow PR. - """ - # Some model config classes like CLIP define their own `from_pretrained` without the new argument `token` yet. - if token is None: - token = kwargs.pop("token", None) - use_auth_token = kwargs.pop("use_auth_token", None) - - if use_auth_token is not None: - warnings.warn( - "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", - FutureWarning, - ) - if token is not None: - raise ValueError( - "`token` and `use_auth_token` are both specified. Please set only the argument `token`." - ) - token = use_auth_token - - if token is not None: - kwargs["token"] = token - def transform(self, X): """ Scikit / Keras interface to transformers' pipelines. This method will forward to __call__(). From 33cc7f568fe03c5a2c621c879d344a38206feb56 Mon Sep 17 00:00:00 2001 From: Hafedh <70411813+not-lain@users.noreply.github.com> Date: Mon, 15 Apr 2024 20:28:13 +0100 Subject: [PATCH 14/14] Update src/transformers/pipelines/base.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> --- src/transformers/pipelines/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index 356b29e705f9ac..1e84c86425870a 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -914,7 +914,6 @@ def save_pretrained( self, save_directory: Union[str, os.PathLike], safe_serialization: bool = True, - token: Optional[Union[str, bool]] = None, **kwargs, ): """