diff --git a/xinference/model/llm/llm_family.py b/xinference/model/llm/llm_family.py
index a25f5ca77b..b3ff856312 100644
--- a/xinference/model/llm/llm_family.py
+++ b/xinference/model/llm/llm_family.py
@@ -171,6 +171,8 @@ def copy(
             raise ValueError(
                 f"Model URI cannot be a relative path: {llm_spec.model_uri}"
             )
+        if not os.path.exists(XINFERENCE_CACHE_DIR):
+            os.makedirs(XINFERENCE_CACHE_DIR, exist_ok=True)
         os.symlink(src_root, cache_dir, target_is_directory=True)
         return cache_dir
     elif src_scheme in SUPPORTED_SCHEMES: