Properly guard PyTorch stuff (huggingface#23452)

* Properly guard PyTorch stuff

* [all-test]

* [all-test] Fix model imports as well

* Making sure StoppingCriteria is always defined

* [all-test]

src/transformers/tools/agents.py

@@ -24,9 +24,8 @@
 import requests
 from huggingface_hub import HfFolder, hf_hub_download, list_spaces
 
-from ..generation import StoppingCriteria, StoppingCriteriaList
-from ..models.auto import AutoModelForCausalLM, AutoTokenizer
-from ..utils import is_openai_available, logging
+from ..models.auto import AutoTokenizer
+from ..utils import is_openai_available, is_torch_available, logging
 from .base import TASK_MAPPING, TOOL_CONFIG_FILE, Tool, load_tool, supports_remote
 from .prompts import CHAT_MESSAGE_PROMPT, CHAT_PROMPT_TEMPLATE, RUN_PROMPT_TEMPLATE
 from .python_interpreter import evaluate
@@ -38,6 +37,12 @@
 if is_openai_available():
     import openai
 
+if is_torch_available():
+    from ..generation import StoppingCriteria, StoppingCriteriaList
+    from ..models.auto import AutoModelForCausalLM
+else:
+    StoppingCriteria = object
+
 _tools_are_initialized = False