FlagAI-Open · marscrazy · May 28, 2022 · May 28, 2022
diff --git a/LICENSE b/LICENSE
@@ -1,3 +1,4 @@
+Copyright © 2022 BAAI. All rights reserved.
 
                                  Apache License
                            Version 2.0, January 2004
@@ -175,28 +176,3 @@
       of your accepting any such warranty or additional liability.
 
    END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
diff --git a/README.md b/README.md
@@ -152,20 +152,13 @@ Thanks for your interest in contributing! There are many ways to get involved;
 start with our [contributor guidelines](CONTRIBUTING.md) and then
 check these [open issues](https://github.com/BAAI-WuDao/Sailing/issues) for specific tasks.
 
+## Contact us
+Scan wechat QR code 
 
-## [License](/LICENSE)
-```
-Copyright [2022] [BAAI]
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+<img src="./flagai_wechat.png" width = "200" height = "200"  align=center />
 
-http://www.apache.org/licenses/LICENSE-2.0
+## [License](/LICENSE)
+The majority of FlagAI is licensed under the [Apache 2.0 license](LICENSE), however portions of the project are available under separate license terms:
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-```
+* Megatron-LM is licensed under the [Megatron-LM license](https://github.com/NVIDIA/Megatron-LM/blob/main/LICENSE)
+* GLM is licensed under the [MIT license](https://github.com/THUDM/GLM/blob/main/LICENSE)
diff --git a/README_zh.md b/README_zh.md
@@ -213,20 +213,14 @@ for text_pair in test_data:
 # 贡献代码
 感谢您对贡献的兴趣！ 参与的方式有很多； 从我们的[贡献者指南](CONTRIBUTING.md) 开始，然后检查这些[未解决的问题](https://github.com/BAAI-WuDao/Sailing/issues)以执行特定任务。
 
+# 联系我们
+欢迎扫码加入FlagAI用户群
 
-# [许可证](/LICENSE)
-```
-Copyright [2022] [BAAI]
+<img src="./flagai_wechat.png" width = "200" height = "200"  align=center />
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
 
-http://www.apache.org/licenses/LICENSE-2.0
+# [许可证](/LICENSE)
+大部分的FlagAI项目是基于[Apache 2.0 license](LICENSE), 但是部分的代码是基于其他的协议:
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-```
+* Megatron-LM 是基于协议[Megatron-LM license](https://github.com/NVIDIA/Megatron-LM/blob/main/LICENSE)
+* GLM 是基于协议[MIT license](https://github.com/THUDM/GLM/blob/main/LICENSE)
diff --git a/examples/bert_title_generation_english/generate.py b/examples/bert_title_generation_english/generate.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import torch
 from flagai.auto_model.auto_loader import AutoLoader
 from flagai.model.predictor.predictor import Predictor

diff --git a/examples/bert_title_generation_english/train.py b/examples/bert_title_generation_english/train.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import os
 import torch
 from torch.utils.data import Dataset

diff --git a/examples/glm_blank_filling/glm_generate_samples.py b/examples/glm_blank_filling/glm_generate_samples.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 # coding=utf-8
 # Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
 #

diff --git a/examples/glm_poetry_generation/generate.py b/examples/glm_poetry_generation/generate.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import torch
 from flagai.auto_model.auto_loader import AutoLoader
 from flagai.model.predictor.predictor import Predictor

diff --git a/examples/glm_poetry_generation/train.py b/examples/glm_poetry_generation/train.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import os
 import torch
 from torch.utils.data import Dataset

diff --git a/examples/glm_pretrain/train.py b/examples/glm_pretrain/train.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.data.tokenizer import GLMLargeChTokenizer
 from flagai.model.glm_model import GLMForSeq2Seq
 from flagai.trainer import Trainer

diff --git a/examples/glm_seq2seq/train.py b/examples/glm_seq2seq/train.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.trainer import Trainer
 from flagai.model.glm_model import GLMForSeq2Seq
 from flagai.data.tokenizer import GLMLargeEnWordPieceTokenizer, GLMLargeChTokenizer

diff --git a/examples/glm_superglue/train.py b/examples/glm_superglue/train.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.trainer import Trainer
 from flagai.model.glm_model import GLMForSingleTokenCloze, GLMForMultiTokenCloze, GLMForMultiTokenClozeFast
 from flagai.data.tokenizer import GLMLargeEnWordPieceTokenizer, GLMLargeChTokenizer

diff --git a/examples/glm_superglue/train_10b_clue.py b/examples/glm_superglue/train_10b_clue.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.trainer import Trainer
 from flagai.model.glm_model import GLMForSingleTokenCloze
 from flagai.data.tokenizer import GLMLargeChTokenizer

diff --git a/examples/glm_superglue/train_10b_superglue.py b/examples/glm_superglue/train_10b_superglue.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.trainer import Trainer
 from flagai.model.glm_model import GLMForSingleTokenCloze
 from flagai.data.tokenizer import GLMLargeEnWordPieceTokenizer

diff --git a/examples/glm_superglue/train_prefix.py b/examples/glm_superglue/train_prefix.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.trainer import Trainer
 from flagai.model.glm_model import GLMForSingleTokenCloze, GLMForMultiTokenCloze, GLMForMultiTokenClozeFast, GLMForSequenceClassification
 from flagai.data.tokenizer import GLMLargeEnWordPieceTokenizer, GLMLargeChTokenizer

diff --git a/examples/glm_title_generation/generate.py b/examples/glm_title_generation/generate.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import torch
 from flagai.auto_model.auto_loader import AutoLoader
 from flagai.model.predictor.predictor import Predictor

diff --git a/examples/glm_title_generation/train.py b/examples/glm_title_generation/train.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import os
 import numpy as np
 import torch

diff --git a/examples/gpt2_text_writting/generate.py b/examples/gpt2_text_writting/generate.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.auto_model.auto_loader import AutoLoader
 from flagai.model.predictor.predictor import Predictor
 

diff --git a/examples/gpt2_title_generation/generate.py b/examples/gpt2_title_generation/generate.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import torch
 from flagai.auto_model.auto_loader import AutoLoader
 from flagai.model.predictor.predictor import Predictor

diff --git a/examples/gpt2_title_generation/train.py b/examples/gpt2_title_generation/train.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import sys
 
 sys.path.append('/data/liuguang/Sailing')

diff --git a/examples/gpt2_title_generation/train_multi_gpu.py b/examples/gpt2_title_generation/train_multi_gpu.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import sys
 
 sys.path.append("/data/liuguang/Sailing")

diff --git a/examples/roberta_faq/1_construct_data.py b/examples/roberta_faq/1_construct_data.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 # construct data
 # data from https://github.com/murufeng/ChineseNlpCorpus
 import torch

diff --git a/examples/roberta_faq/2_test_bert_faq.py b/examples/roberta_faq/2_test_bert_faq.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import faiss
 import torch
 from flagai.auto_model.auto_loader import AutoLoader

diff --git a/examples/roberta_ner/generate.py b/examples/roberta_ner/generate.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import torch
 from flagai.auto_model.auto_loader import AutoLoader
 from flagai.model.predictor.predictor import Predictor

diff --git a/examples/roberta_ner/generate_crf.py b/examples/roberta_ner/generate_crf.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import torch
 from flagai.auto_model.auto_loader import AutoLoader
 from flagai.model.predictor.predictor import Predictor

diff --git a/examples/roberta_ner/generate_global_pointer.py b/examples/roberta_ner/generate_global_pointer.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import torch
 from flagai.auto_model.auto_loader import AutoLoader
 from flagai.model.predictor.predictor import Predictor

diff --git a/examples/roberta_ner/train.py b/examples/roberta_ner/train.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import os
 
 from tqdm import tqdm

diff --git a/examples/roberta_ner/train_crf.py b/examples/roberta_ner/train_crf.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 from tqdm import tqdm
 import torch
 from torch.utils.data import Dataset

diff --git a/examples/roberta_ner/train_global_pointer.py b/examples/roberta_ner/train_global_pointer.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 from tqdm import tqdm
 import torch
 from torch.utils.data import Dataset

diff --git a/examples/roberta_semantic_matching/generate.py b/examples/roberta_semantic_matching/generate.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import torch
 from flagai.auto_model.auto_loader import AutoLoader
 from flagai.model.predictor.predictor import Predictor

diff --git a/examples/roberta_semantic_matching/train.py b/examples/roberta_semantic_matching/train.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import os
 import torch
 from torch.utils.data import Dataset

diff --git a/examples/roberta_title_generation/generate.py b/examples/roberta_title_generation/generate.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import torch
 from flagai.auto_model.auto_loader import AutoLoader
 from flagai.model.predictor.predictor import Predictor

diff --git a/examples/roberta_title_generation/train.py b/examples/roberta_title_generation/train.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import os
 import torch
 from torch.utils.data import Dataset

diff --git a/examples/roberta_title_generation/train_multi_gpu.py b/examples/roberta_title_generation/train_multi_gpu.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import sys
 
 sys.path.append("/data/liuguang/Sailing")

diff --git a/examples/t5_flagai_11b/deepspeed.json b/examples/t5_flagai_11b/deepspeed.json
@@ -0,0 +1,40 @@
+{
+    "train_micro_batch_size_per_gpu": 2,
+    "gradient_accumulation_steps": 1,
+    "steps_per_print": 100,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+      "stage": 3,
+      "contiguous_gradients": false,
+      "overlap_comm": true,
+      "reduce_scatter": true,
+      "reduce_bucket_size": 5e7,
+      "allgather_bucket_size": 5e7,
+      "cpu_offload": true 
+    },
+    "zero_allow_untested_optimizer": true,
+    "fp16": {
+      "enabled": true,
+      "loss_scale": 0,
+      "loss_scale_window": 1000,
+      "hysteresis": 2,
+      "min_loss_scale": 1
+    },
+    "optimizer": {
+      "type": "Adam",
+      "params": {
+        "lr": 0.0004,
+        "weight_decay": 0.01,
+        "betas": [
+          0.9,
+          0.98
+        ],
+        "eps": 1e-6
+      }
+    },
+    "activation_checkpointing": {
+      "partition_activations": false,
+      "contiguous_memory_optimization": false
+    },
+    "wall_clock_breakdown": false
+  }
diff --git a/examples/t5_flagai_11b/train_title_with_flagai_t5_11b.py b/examples/t5_flagai_11b/train_title_with_flagai_t5_11b.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import sys
 
 sys.path.append('/mnt/liuguang/FlagAI')
@@ -74,7 +77,7 @@ def read_file():
 
 model = T5ForConditionalGeneration.from_pretrain(download_path='/mnt',
                                                  model_name='t5-11b')
-
+model.gradient_checkpointing = True
 print("loading model & tokenizer is done!")
 
 maxlen = 1024

diff --git a/examples/t5_huggingface/train_t5_11b.py b/examples/t5_huggingface/train_t5_11b.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.trainer import Trainer
 from transformers import T5ForConditionalGeneration, T5Tokenizer
 from torch.utils.data import Dataset

diff --git a/examples/t5_title_generation/generate.py b/examples/t5_title_generation/generate.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.auto_model.auto_loader import AutoLoader
 from flagai.model.predictor.predictor import Predictor
 

diff --git a/examples/t5_title_generation/train.py b/examples/t5_title_generation/train.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import os
 import torch
 from torch.utils.data import Dataset

diff --git a/flagai/auto_model/auto_loader.py b/flagai/auto_model/auto_loader.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import importlib
 import os
 from  flagai.model.file_utils import _get_model_id, _get_vocab_path

diff --git a/flagai/data/collate_utils.py b/flagai/data/collate_utils.py
@@ -1,3 +1,6 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License")
 import numpy as np
 import torch