Merge branch 'master' into lyj/fix_falcon40b

.github/workflows/nv-a6000.yml

@@ -3,8 +3,9 @@ name: nv-a6000
 on:
  pull_request:
  paths:
- - "deepspeed/inference/v2/**"
- - "tests/unit/inference/v2/**"
+ - 'deepspeed/inference/v2/**'
+ - 'tests/unit/inference/v2/**'
+ - '.github/workflows/nv-a6000.yml'
  workflow_dispatch:
 
 concurrency:

.github/workflows/nv-accelerate-v100.yml

@@ -6,7 +6,7 @@ on:
  - 'docs/**'
  - 'blogs/**'
  - 'deepspeed/inference/v2/**'
- - "tests/unit/inference/v2/**"
+ - 'tests/unit/inference/v2/**'
  merge_group:
  branches: [ master ]
  schedule:

.github/workflows/nv-inference.yml

@@ -6,7 +6,7 @@ on:
  - 'docs/**'
  - 'blogs/**'
  - 'deepspeed/inference/v2/**'
- - "tests/unit/inference/v2/**"
+ - 'tests/unit/inference/v2/**'
  merge_group:
  branches: [ master ]
  schedule:

.github/workflows/nv-lightning-v100.yml

@@ -6,7 +6,7 @@ on:
  - 'docs/**'
  - 'blogs/**'
  - 'deepspeed/inference/v2/**'
- - "tests/unit/inference/v2/**"
+ - 'tests/unit/inference/v2/**'
  merge_group:
  branches: [ master ]
  schedule:

.github/workflows/nv-megatron.yml

@@ -6,7 +6,7 @@ on:
  - 'docs/**'
  - 'blogs/**'
  - 'deepspeed/inference/v2/**'
- - "tests/unit/inference/v2/**"
+ - 'tests/unit/inference/v2/**'
  merge_group:
  branches: [ master ]
  schedule:

.github/workflows/nv-pre-compile-ops.yml

@@ -8,7 +8,7 @@ on:
  - 'docs/**'
  - 'blogs/**'
  - 'deepspeed/inference/v2/**'
- - "tests/unit/inference/v2/**"
+ - 'tests/unit/inference/v2/**'
  merge_group:
  branches: [ master ]
  schedule:
@@ -19,7 +19,7 @@ concurrency:
  cancel-in-progress: true
 
 jobs:
- build-ops:
+ unit-tests:
  runs-on: ubuntu-20.04
  container:
  image: deepspeed/gh-builder:ubuntu1804-py38-torch1131-cu116

.github/workflows/nv-sd.yml

@@ -10,6 +10,7 @@ on:
  - "tests/unit/inference/test_stable_diffusion.py"
  - "deepspeed/model_implementations/diffusers/unet.py"
  - "deepspeed/model_implementations/diffusers/vae.py"
+ - ".github/workflows/nv-sd.yml"
 
 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/nv-torch-latest-cpu.yml

@@ -6,7 +6,7 @@ on:
  - 'docs/**'
  - 'blogs/**'
  - 'deepspeed/inference/v2/**'
- - "tests/unit/inference/v2/**"
+ - 'tests/unit/inference/v2/**'
  merge_group:
  branches: [ master ]
  schedule:

.github/workflows/nv-torch-latest-v100.yml

@@ -6,7 +6,7 @@ on:
  - 'docs/**'
  - 'blogs/**'
  - 'deepspeed/inference/v2/**'
- - "tests/unit/inference/v2/**"
+ - 'tests/unit/inference/v2/**'
  merge_group:
  branches: [ master ]
  schedule:

.github/workflows/nv-transformers-v100.yml

@@ -6,7 +6,7 @@ on:
  - 'docs/**'
  - 'blogs/**'
  - 'deepspeed/inference/v2/**'
- - "tests/unit/inference/v2/**"
+ - 'tests/unit/inference/v2/**'
  merge_group:
  branches: [ master ]
  schedule:

.gitignore

@@ -1,40 +1,55 @@
-# Ignore Python compiled files
+## Ignore Python compiled files
 *.pyc
 
-# Ignore IDE-specific files and directories
-.idea/ # JetBrains IDE settings
-.vscode/ # Visual Studio Code settings
-.theia/ # Theia IDE settings
-
-# Ignore temporary and backup files
-*~ # General backup files
-*.swp # Vim swap files
-
-# Ignore log files
+## Ignore IDE-specific files and directories
+# JetBrains IDE settings
+.idea/
+# Visual Studio Code settings
+.vscode/
+# Theia IDE settings
+.theia/
+
+## Ignore temporary and backup files
+# General backup files
+*~
+# Vim swap files
+*.swp
+
+## Ignore log files
 *.log
 
-# Ignore a specific generated file
+## Ignore a specific generated file
 deepspeed/git_version_info_installed.py
 
-# Ignore Python bytecode cache
+## Ignore Python bytecode cache
 __pycache__
 
-# Build + installation data
-build/ # Build artifacts
-dist/ # Distribution files
-*.so # Compiled shared objects
-deepspeed.egg-info/ # Deepspeed package info
-build.txt # Build information
-
-# Website generated files
-docs/_site/ # Jekyll generated site
-docs/build # Generated documentation
+## Build + installation data
+# Build artifacts
+build/
+# Distribution files
+dist/
+# Compiled shared objects
+*.so
+# Deepspeed package info
+deepspeed.egg-info/
+# Build information
+build.txt
+
+## Website generated files
+# Jekyll generated site
+docs/_site/
+# Generated documentation
+docs/build
 docs/code-docs/source/_build
 docs/code-docs/_build
 docs/code-docs/build
-.sass-cache/ # SASS cache
-.jekyll-cache/ # Jekyll cache
+# SASS cache
+.sass-cache/
+# Jekyll cache
+.jekyll-cache/
 .jekyll-metadata
 
-# Testing data
-tests/unit/saved_checkpoint/ # Saved checkpoints for testing
+## Testing data
+# Saved checkpoints for testing
+tests/unit/saved_checkpoint/

README.md

@@ -15,6 +15,7 @@
 ## Latest News
 <b> <span style="color:orange" > DeepSpeed empowers ChatGPT-like model training with a single click, offering 15x speedup over SOTA RLHF systems with unprecedented cost reduction at all scales; [learn how](https://github.com/microsoft/DeepSpeed/tree/master/blogs/deepspeed-chat)</span>.</b>
 
+* [2023/11] [Llama 2 Inference on 4th Gen Intel® Xeon® Scalable Processor with DeepSpeed](https://github.com/microsoft/DeepSpeed/tree/master/blogs/intel-inference) [[Intel version]](https://www.intel.com/content/www/us/en/developer/articles/technical/xllama-2-on-xeon-scalable-processor-with-deepspeed.html)
 * [2023/11] [DeepSpeed ZeRO-Offload++: 6x Higher Training Throughput via Collaborative CPU/GPU Twin-Flow](https://github.com/microsoft/DeepSpeed/tree/master/blogs/deepspeed-offloadpp)
 * [2023/11] [DeepSpeed-FastGen: High-throughput Text Generation for LLMs via MII and DeepSpeed-Inference](https://github.com/microsoft/DeepSpeed/tree/master/blogs/deepspeed-fastgen) [[English](https://github.com/microsoft/DeepSpeed/tree/master/blogs/deepspeed-fastgen)] [[中文](https://github.com/microsoft/DeepSpeed/tree/master/blogs/deepspeed-fastgen/chinese/README.md)] [[日本語](https://github.com/microsoft/DeepSpeed/tree/master/blogs/deepspeed-fastgen/japanese/README.md)]
 * [2023/10] [DeepSpeed-VisualChat: Improve Your Chat Experience with Multi-Round Multi-Image Inputs](https://github.com/microsoft/DeepSpeed/tree/master/blogs/deepspeed-visualchat/10-03-2023/README.md) [[English](https://github.com/microsoft/DeepSpeed/tree/master/blogs/deepspeed-visualchat/10-03-2023/README.md)] [[中文](https://github.com/microsoft/DeepSpeed/blob/master/blogs/deepspeed-visualchat/10-03-2023/README-Chinese.md)] [[日本語](https://github.com/microsoft/DeepSpeed/blob/master/blogs/deepspeed-visualchat/10-03-2023/README-Japanese.md)]
@@ -128,7 +129,7 @@ DeepSpeed has been integrated with several different popular open-source DL fram
 | AMD | [![amd-mi100](https://github.com/microsoft/DeepSpeed/actions/workflows/amd-mi100.yml/badge.svg?branch=master)](https://github.com/microsoft/DeepSpeed/actions/workflows/amd-mi100.yml) [![amd-mi200](https://github.com/microsoft/DeepSpeed/actions/workflows/amd-mi200.yml/badge.svg?branch=master)](https://github.com/microsoft/DeepSpeed/actions/workflows/amd-mi200.yml) |
 | CPU | [![nv-torch-latest-cpu](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-torch-latest-cpu.yml/badge.svg?branch=master)](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-torch-latest-cpu.yml) |
 | PyTorch Nightly | [![nv-torch-nightly-v100](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-torch-nightly-v100.yml/badge.svg?branch=master)](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-torch-nightly-v100.yml) |
-| Integrations | [![nv-transformers-v100](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-transformers-v100.yml/badge.svg?branch=master)](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-transformers-v100.yml) [![nv-lightning-v100](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-lightning-v100.yml/badge.svg?branch=master)](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-lightning-v100.yml) [![nv-accelerate-v100](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-accelerate-v100.yml/badge.svg?branch=master)](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-accelerate-v100.yml) [![nv-megatron](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-megatron.yml/badge.svg?branch=master)](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-megatron.yml) [![nv-mii](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-mii.yml/badge.svg?branch=master)](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-mii.yml) [![nv-ds-chat](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-ds-chat.yml/badge.svg?branch=master)](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-ds-chat.yml) |
+| Integrations | [![nv-transformers-v100](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-transformers-v100.yml/badge.svg?branch=master)](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-transformers-v100.yml) [![nv-lightning-v100](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-lightning-v100.yml/badge.svg?branch=master)](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-lightning-v100.yml) [![nv-accelerate-v100](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-accelerate-v100.yml/badge.svg?branch=master)](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-accelerate-v100.yml) [![nv-megatron](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-megatron.yml/badge.svg?branch=master)](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-megatron.yml) [![nv-mii](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-mii.yml/badge.svg?branch=master)](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-mii.yml) [![nv-ds-chat](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-ds-chat.yml/badge.svg?branch=master)](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-ds-chat.yml) [![nv-sd](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-sd.yml/badge.svg)](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-sd.yml) |
 | Misc | [![Formatting](https://github.com/microsoft/DeepSpeed/actions/workflows/formatting.yml/badge.svg?branch=master)](https://github.com/microsoft/DeepSpeed/actions/workflows/formatting.yml) [![pages-build-deployment](https://github.com/microsoft/DeepSpeed/actions/workflows/pages/pages-build-deployment/badge.svg)](https://github.com/microsoft/DeepSpeed/actions/workflows/pages/pages-build-deployment) [![Documentation Status](https://readthedocs.org/projects/deepspeed/badge/?version=latest)](https://deepspeed.readthedocs.io/en/latest/?badge=latest)[![python](https://github.com/microsoft/DeepSpeed/actions/workflows/python.yml/badge.svg?branch=master)](https://github.com/microsoft/DeepSpeed/actions/workflows/python.yml) |
 
 # Installation

accelerator/cpu_accelerator.py

@@ -63,7 +63,7 @@ def random(self):
  return torch.random
 
  def set_rng_state(self, new_state, device_index=None):
- if device_index == None:
+ if device_index is None:
  return torch.set_rng_state(new_state)
  return torch.set_rng_state(new_state, device_index)
 
@@ -253,7 +253,7 @@ def on_accelerator(self, tensor):
  # create an instance of op builder and return, name specified by class_name
  def create_op_builder(self, op_name):
  builder_class = self.get_op_builder(op_name)
- if builder_class != None:
+ if builder_class is not None:
  return builder_class()
  return None
 

accelerator/cuda_accelerator.py

@@ -44,7 +44,7 @@ def is_synchronized_device(self):
 
  # Device APIs
  def device_name(self, device_index=None):
- if device_index == None:
+ if device_index is None:
  return 'cuda'
  return 'cuda:{}'.format(device_index)
 
@@ -280,7 +280,7 @@ def op_builder_dir(self):
  class_dict = None
 
  def _lazy_init_class_dict(self):
- if self.class_dict != None:
+ if self.class_dict is not None:
  return
  else:
  self.class_dict = {}

accelerator/mps_accelerator.py

@@ -26,7 +26,7 @@ def is_synchronized_device(self):
 
  # Device APIs
  def device_name(self, device_index=None):
- if device_index == None:
+ if device_index is None:
  return "mps"
  return "mps:{}".format(device_index)
 
@@ -221,7 +221,7 @@ def op_builder_dir(self):
  # create an instance of op builder, specified by class_name
  def create_op_builder(self, op_name):
  builder_class = self.get_op_builder(op_name)
- if builder_class != None:
+ if builder_class is not None:
  return builder_class()
  return None
 

accelerator/npu_accelerator.py

@@ -30,7 +30,7 @@ def is_synchronized_device(self):
 
  # Device APIs
  def device_name(self, device_index=None):
- if device_index == None:
+ if device_index is None:
  return 'npu'
  return 'npu:{}'.format(device_index)
 

accelerator/real_accelerator.py

@@ -45,7 +45,7 @@ def _validate_accelerator(accel_obj):
 
 
 def is_current_accelerator_supported():
- return get_accelerator() in SUPPORTED_ACCELERATOR_LIST
+ return get_accelerator().device_name() in SUPPORTED_ACCELERATOR_LIST
 
 
 def get_accelerator():