From d1ea7933ef725f24a38ba582c8dc8b1f70e97869 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Tue, 18 Jul 2023 04:05:24 -0700
Subject: [PATCH 01/52] deepspeed

---
 Pipfile                                       |   2 +
 Pipfile.lock                                  | 496 ++++++++++++------
 app_utils/utils.py                            |  50 +-
 .../text_causal_language_modeling_config.py   |   1 +
 llm_studio/src/utils/modeling_utils.py        | 170 ++++--
 train.py                                      |  48 +-
 6 files changed, 525 insertions(+), 242 deletions(-)

diff --git a/Pipfile b/Pipfile
index a99f31168e..eaefd3158a 100644
--- a/Pipfile
+++ b/Pipfile
@@ -49,6 +49,8 @@ Jinja2 = ">=3.1.2, <4.0.0"
 tenacity = ">=8.2.2, <9.0.0"
 h2o-wave = "0.26"
 tiktoken = "*"
+deepspeed = ">=0.9.2"
+mpi4py = ">=3.1.4"
 
 [dev-packages]
 pytest = "==7.1.3"
diff --git a/Pipfile.lock b/Pipfile.lock
index 7fa2747917..5bd56dc5a7 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "523dde3cf862e0310b13b878e9b5a0d08670c1c584c549811a50a36a3e9c06fd"
+            "sha256": "196b33e3d862a1c80dd953eefcdd066965c4a8b03017a41a6a66b3c09ea35ba2"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -132,11 +132,11 @@
         },
         "anyio": {
             "hashes": [
-                "sha256:275d9973793619a5374e1c89a4f4ad3f4b0a5510a2b5b939444bee8f4c4d37ce",
-                "sha256:eddca883c4175f14df8aedce21054bfca3adb70ffe76a9f607aef9d7fa2ea7f0"
+                "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780",
+                "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.7.0"
+            "version": "==3.7.1"
         },
         "arrow": {
             "hashes": [
@@ -162,6 +162,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==23.1.0"
         },
+        "backoff": {
+            "hashes": [
+                "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba",
+                "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"
+            ],
+            "markers": "python_version >= '3.7' and python_version < '4.0'",
+            "version": "==2.2.1"
+        },
         "beautifulsoup4": {
             "hashes": [
                 "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30",
@@ -178,6 +186,14 @@
             "index": "pypi",
             "version": "==0.39.1"
         },
+        "bleach": {
+            "hashes": [
+                "sha256:1a1a85c1595e07d8db14c5f09f09e6433502c51c595970edc090551f0db99414",
+                "sha256:33c16e3353dbd13028ab4799a0f89a83f113405c766e9c122df8a06f5b85b3f4"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==6.0.0"
+        },
         "blessed": {
             "hashes": [
                 "sha256:0c542922586a265e699188e52d5f5ac5ec0dd517e5a1041d90d2bbf23f906058",
@@ -317,11 +333,11 @@
         },
         "click": {
             "hashes": [
-                "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e",
-                "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"
+                "sha256:2739815aaa5d2c986a88f1e9230c55e17f0caad3d958a5e13ad0797c166db9e3",
+                "sha256:b97d0c74955da062a7d4ef92fadb583806a585b2ea81958a81bd72726cbb8e37"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.1.3"
+            "version": "==8.1.4"
         },
         "cmake": {
             "hashes": [
@@ -434,6 +450,13 @@
             "index": "pypi",
             "version": "==2.11.0"
         },
+        "deepspeed": {
+            "hashes": [
+                "sha256:a3743db45066afb67fde562e586e92ee5afd8f5ab4c390e6f796f5ec1b112290"
+            ],
+            "index": "pypi",
+            "version": "==0.9.5"
+        },
         "dill": {
             "hashes": [
                 "sha256:7e40e4a70304fd9ceab3535d36e58791d9c4a776b38ec7f7ec9afc8d3dca4d4f",
@@ -442,12 +465,6 @@
             "index": "pypi",
             "version": "==0.3.4"
         },
-        "docopt": {
-            "hashes": [
-                "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"
-            ],
-            "version": "==0.6.2"
-        },
         "einops": {
             "hashes": [
                 "sha256:99149e46cc808956b174932fe563d920db4d6e5dadb8c6ecdaa7483b7ef7cfc3",
@@ -458,11 +475,11 @@
         },
         "exceptiongroup": {
             "hashes": [
-                "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e",
-                "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"
+                "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5",
+                "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"
             ],
             "markers": "python_version < '3.11'",
-            "version": "==1.1.1"
+            "version": "==1.1.2"
         },
         "fastparquet": {
             "hashes": [
@@ -714,23 +731,30 @@
         },
         "h2o-wave": {
             "hashes": [
-                "sha256:4445414ba444dc1d4f98d3bda0244e33b27cc4edf5f217d284c14d8cf99e3ae6",
-                "sha256:673045e090322eda3132fb5106dabe3c519136b6b4a0ef963ea7c7519a08b503",
-                "sha256:6cddd853084097aaf80b8ec3a92d21c944c8e7aadbf69b012c96c7147bc7a5b7",
-                "sha256:7c5bbeed1b7140dac2efaed62a50b457854719f939f7b787d9157e316eb82540",
-                "sha256:9f55dd9f122930524d32655fe469a14e4e1748c07846a0b58e337b4ca612b6a4",
-                "sha256:f317360554c9769968ece50033b71ad5639dde3dc66a8b2a76ac199ca15426bc"
+                "sha256:4b749ae79558f9d9cf5b6fa4ea1e06a4ba4353976a5c48d1536d26feed9b4d12",
+                "sha256:591160922dc67023a9233611a5708d830dce919a4069f7a2c7f4e5b27d6d9bcb",
+                "sha256:5ea8843f97636e190ac36437c978db00ba38911838d839de52d2405877a7cb71",
+                "sha256:86e118991de4dc441ac22a863f97a7ec3a1f897ea41fc955e2da96ec5c1ad6d7",
+                "sha256:ef952af629f9fb8c9c2b4adceb0deb4bbd0d35de5097e1238bfc203129bfe870",
+                "sha256:f8ddd0a69da26963302502591166c2302d9eba4a0ed63916052c5b6c4849f6ac"
             ],
             "index": "pypi",
-            "version": "==0.26.0"
+            "version": "==0.26.1"
+        },
+        "hjson": {
+            "hashes": [
+                "sha256:55af475a27cf83a7969c808399d7bccdec8fb836a07ddbd574587593b9cdcf75",
+                "sha256:65713cdcf13214fb554eb8b4ef803419733f4f5e551047c9b711098ab7186b89"
+            ],
+            "version": "==3.1.0"
         },
         "httpcore": {
             "hashes": [
-                "sha256:125f8375ab60036db632f34f4b627a9ad085048eef7cb7d2616fea0f739f98af",
-                "sha256:5581b9c12379c4288fe70f43c710d16060c10080617001e6b22a3b6dbcbefd36"
+                "sha256:a6f30213335e34c1ade7be6ec7c47f19f50c56db36abef1a9dfa3815b1cb3888",
+                "sha256:c2789b767ddddfa2a5782e3199b2b7f6894540b17b16ec26b2c4d8e103510b87"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.17.2"
+            "version": "==0.17.3"
         },
         "httpx": {
             "hashes": [
@@ -789,11 +813,11 @@
         },
         "joblib": {
             "hashes": [
-                "sha256:0b12a65dc76c530dbd790dd92881f75c40932b4254a7c8e608a868df408ca0a3",
-                "sha256:172d56d4c43dd6bcd953bea213018c4084cf754963bbf54b8dae40faea716b98"
+                "sha256:1f937906df65329ba98013dc9692fe22a4c5e4a648112de500508b18a21b41e3",
+                "sha256:89cf0529520e01b3de7ac7b74a8102c90d16d54c64b5dd98cafcd14307fdf915"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.3.0"
+            "version": "==1.3.1"
         },
         "jsonpointer": {
             "hashes": [
@@ -812,18 +836,26 @@
         },
         "jsonschema": {
             "hashes": [
-                "sha256:0f864437ab8b6076ba6707453ef8f98a6a0d512a80e93f8abdb676f737ecb60d",
-                "sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6"
+                "sha256:8caf5b57a990a98e9b39832ef3cb35c176fe331414252b6e1b26fd5866f891a4",
+                "sha256:b508dd6142bd03f4c3670534c80af68cd7bbff9ea830b9cf2625d4a3c49ddf60"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==4.17.3"
+            "markers": "python_version >= '3.8'",
+            "version": "==4.18.0"
+        },
+        "jsonschema-specifications": {
+            "hashes": [
+                "sha256:3d2b82663aff01815f744bb5c7887e2121a63399b49b104a3c96145474d091d7",
+                "sha256:ca1c4dd059a9e7b34101cf5b3ab7ff1d18b139f35950d598d629837ef66e8f28"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==2023.6.1"
         },
         "kaggle": {
             "hashes": [
-                "sha256:8364c56c36125cb8195876d911d0bc9efbc173167e963b9e9ef793789ba9a7fe"
+                "sha256:fa29950f04bf6712e76e9f6fea5b9a3ee03bc971f63abb6a3df395d89a691d02"
             ],
             "index": "pypi",
-            "version": "==1.5.13"
+            "version": "==1.5.15"
         },
         "lit": {
             "hashes": [
@@ -894,6 +926,29 @@
             ],
             "version": "==1.6"
         },
+        "mpi4py": {
+            "hashes": [
+                "sha256:152e68aeae40279ca435d15487303159fe3d229337f204141ea805c67b4fbbc3",
+                "sha256:17858f2ebc623220d0120d1fa8d428d033dde749c4bc35b33d81a66ad7f93480",
+                "sha256:1f5f5560211e5b02c388a4e9521acdc5c6ab076e18a60109dc4bcbdb26f0e506",
+                "sha256:32b46ed9d4691ea790d77dde6c5489abd6a4cf6f8e61c35871924c2170bcb7fb",
+                "sha256:4fac4ea098f877ac2d53e6c08f12e84161b960238725ec65795cb4ca391a0649",
+                "sha256:56c79531fc9af9a099c413f38efb20503a84b8355d72a63f0297399f474d88c9",
+                "sha256:5785eecf65669fa97e258d087a86e5b2c5e4dd2b5140e9aee14917cadfe3c876",
+                "sha256:6721a23a2a8ef5a2841ba64926e8e4be0a559344557eef0b2e81821283d56fd1",
+                "sha256:6a7c9dfd1d051aeb1c5c9cc37d8997701732e2e1780f4e188c29c443f6feb6b8",
+                "sha256:6ad6318092adc2bda1567b651d8977f2e418312557cda900805f929f6ccd93b9",
+                "sha256:878ee8c66f3cb4aced2d9d6bde2bbcb5d17d1f90ad0b7407f57e993f6c849a16",
+                "sha256:9e3075e06cea9556cae6562d9194d2c6278c6c8f66af36d8094fb2f77392a90f",
+                "sha256:9ed1b189f859337acc49c2a12a7e13cb2ef3309ce3486b986b082078c4e7493e",
+                "sha256:a0c7f5cee1ffc19d0d145b24eb06997962fe0a07e3019abf5e1575fb43c4ac03",
+                "sha256:bc102ec71939bff487af247f727216a05c45930d931db3f5b206fad9bfcba9c0",
+                "sha256:d939c895ae97256e80ef6e6d626f51bea187d68ab54dd772971fb134632e7ee6",
+                "sha256:f37859af5a318d1af07d3b51714fd36b7674e7368191578cfd96165136279f2d"
+            ],
+            "index": "pypi",
+            "version": "==3.1.4"
+        },
         "mpmath": {
             "hashes": [
                 "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f",
@@ -1069,11 +1124,11 @@
         },
         "neptune": {
             "hashes": [
-                "sha256:be5ea84ea10309d7a44b5a0b899f08319a1ac1ee727460efb9c549ee36e4584d",
-                "sha256:d78cf6a74e23b679207e2ea64de72a720a5b12a007059c7277c6be6d5cc283b7"
+                "sha256:8e2d08a9282696fc6d1c52f3309d79daae366a8f9dc71c0f9d10177376d30d78",
+                "sha256:f4ea30fa6312659c86718e5116f71b9611372b741092a50c74422ba4db161550"
             ],
             "index": "pypi",
-            "version": "==1.3.1"
+            "version": "==1.3.2"
         },
         "networkx": {
             "hashes": [
@@ -1083,6 +1138,28 @@
             "markers": "python_version >= '3.8'",
             "version": "==3.1"
         },
+        "ninja": {
+            "hashes": [
+                "sha256:1c474326e11fba3f8c2582715d79216292e327d3335367c0e87e9647a002cc4a",
+                "sha256:3329b4b7c1694730772522a3ba0ba40fd15c012476ed3e1c9f0fd9e76190394e",
+                "sha256:34753459493543782d87267e4cad63dd4639b07f8394ffe6d4417e9eda05c8a8",
+                "sha256:3b28b595ed580752240ade7821b6cb7a5a4c6a604c865dc474bd38f06e2eb7f5",
+                "sha256:4e547bc759c570773d83d110c41fd5ca9a94c0a9a8388f5a3ea37bdf97d002b0",
+                "sha256:60179bb4f22c88279c53a5402bb5fe81c97c627a28d93c737d1fa067d892115d",
+                "sha256:642cb64d859276998f14972724850e0c5b7febbc1bce3d2065b7e0cb7d3a0b79",
+                "sha256:6f6465a7efe6473a2a34edab83633594de19d59406a727316e1367ebcc528908",
+                "sha256:779f228e407c54a8b6e4cbf8f835489998dd250f67bf1b9bd7b8a8ab6bdcdc7b",
+                "sha256:817e2aee2a4d28a708a67bcfba1817ae502c32c6d8ef80e50d63b0f23adf3a08",
+                "sha256:8cf96f92ccc851c600cb3e1251c34db06f1dd682de79188ad490c33cddc66981",
+                "sha256:a7a564fe755ddfbdbccb07b0b758e3f8460e5f8ba1adaab40a5eaa2f8c01ce68",
+                "sha256:ba50a32424912e5f3ee40d791b506a160dc0eeda7de5ad8faebe7aa8006244dc",
+                "sha256:c833a47d39b2d1eee3f9ca886fa1581efd5be6068b82734ac229961ee8748f90",
+                "sha256:df11b8afea0501883e33faeb1c43d2ef67f466d5f4bd85f9c376e9a93a43a277",
+                "sha256:edec1053e141253076b2df7ec03a246ff581e9270aa1ca9759397b21e2760e57",
+                "sha256:f48c3c6eea204062f6bbf089dfc63e1ad41a08640e1da46ef2b30fa426f7ce23"
+            ],
+            "version": "==1.11.1"
+        },
         "numpy": {
             "hashes": [
                 "sha256:0ac6edfb35d2a99aaf102b509c8e9319c499ebd4978df4971b94419a116d0790",
@@ -1173,87 +1250,67 @@
         },
         "peft": {
             "git": "https://github.com/huggingface/peft",
-            "ref": "86290e9660d24ef0d0cedcf57710da249dd1f2f4"
+            "ref": "39ef2546d5d9b8f5f8a7016ec10657887a867041"
         },
         "pillow": {
             "hashes": [
-                "sha256:07999f5834bdc404c442146942a2ecadd1cb6292f5229f4ed3b31e0a108746b1",
-                "sha256:0852ddb76d85f127c135b6dd1f0bb88dbb9ee990d2cd9aa9e28526c93e794fba",
-                "sha256:1781a624c229cb35a2ac31cc4a77e28cafc8900733a864870c49bfeedacd106a",
-                "sha256:1e7723bd90ef94eda669a3c2c19d549874dd5badaeefabefd26053304abe5799",
-                "sha256:229e2c79c00e85989a34b5981a2b67aa079fd08c903f0aaead522a1d68d79e51",
-                "sha256:22baf0c3cf0c7f26e82d6e1adf118027afb325e703922c8dfc1d5d0156bb2eeb",
-                "sha256:252a03f1bdddce077eff2354c3861bf437c892fb1832f75ce813ee94347aa9b5",
-                "sha256:2dfaaf10b6172697b9bceb9a3bd7b951819d1ca339a5ef294d1f1ac6d7f63270",
-                "sha256:322724c0032af6692456cd6ed554bb85f8149214d97398bb80613b04e33769f6",
-                "sha256:35f6e77122a0c0762268216315bf239cf52b88865bba522999dc38f1c52b9b47",
-                "sha256:375f6e5ee9620a271acb6820b3d1e94ffa8e741c0601db4c0c4d3cb0a9c224bf",
-                "sha256:3ded42b9ad70e5f1754fb7c2e2d6465a9c842e41d178f262e08b8c85ed8a1d8e",
-                "sha256:432b975c009cf649420615388561c0ce7cc31ce9b2e374db659ee4f7d57a1f8b",
-                "sha256:482877592e927fd263028c105b36272398e3e1be3269efda09f6ba21fd83ec66",
-                "sha256:489f8389261e5ed43ac8ff7b453162af39c3e8abd730af8363587ba64bb2e865",
-                "sha256:54f7102ad31a3de5666827526e248c3530b3a33539dbda27c6843d19d72644ec",
-                "sha256:560737e70cb9c6255d6dcba3de6578a9e2ec4b573659943a5e7e4af13f298f5c",
-                "sha256:5671583eab84af046a397d6d0ba25343c00cd50bce03787948e0fff01d4fd9b1",
-                "sha256:5ba1b81ee69573fe7124881762bb4cd2e4b6ed9dd28c9c60a632902fe8db8b38",
-                "sha256:5d4ebf8e1db4441a55c509c4baa7a0587a0210f7cd25fcfe74dbbce7a4bd1906",
-                "sha256:60037a8db8750e474af7ffc9faa9b5859e6c6d0a50e55c45576bf28be7419705",
-                "sha256:608488bdcbdb4ba7837461442b90ea6f3079397ddc968c31265c1e056964f1ef",
-                "sha256:6608ff3bf781eee0cd14d0901a2b9cc3d3834516532e3bd673a0a204dc8615fc",
-                "sha256:662da1f3f89a302cc22faa9f14a262c2e3951f9dbc9617609a47521c69dd9f8f",
-                "sha256:7002d0797a3e4193c7cdee3198d7c14f92c0836d6b4a3f3046a64bd1ce8df2bf",
-                "sha256:763782b2e03e45e2c77d7779875f4432e25121ef002a41829d8868700d119392",
-                "sha256:77165c4a5e7d5a284f10a6efaa39a0ae8ba839da344f20b111d62cc932fa4e5d",
-                "sha256:7c9af5a3b406a50e313467e3565fc99929717f780164fe6fbb7704edba0cebbe",
-                "sha256:7ec6f6ce99dab90b52da21cf0dc519e21095e332ff3b399a357c187b1a5eee32",
-                "sha256:833b86a98e0ede388fa29363159c9b1a294b0905b5128baf01db683672f230f5",
-                "sha256:84a6f19ce086c1bf894644b43cd129702f781ba5751ca8572f08aa40ef0ab7b7",
-                "sha256:8507eda3cd0608a1f94f58c64817e83ec12fa93a9436938b191b80d9e4c0fc44",
-                "sha256:85ec677246533e27770b0de5cf0f9d6e4ec0c212a1f89dfc941b64b21226009d",
-                "sha256:8aca1152d93dcc27dc55395604dcfc55bed5f25ef4c98716a928bacba90d33a3",
-                "sha256:8d935f924bbab8f0a9a28404422da8af4904e36d5c33fc6f677e4c4485515625",
-                "sha256:8f36397bf3f7d7c6a3abdea815ecf6fd14e7fcd4418ab24bae01008d8d8ca15e",
-                "sha256:91ec6fe47b5eb5a9968c79ad9ed78c342b1f97a091677ba0e012701add857829",
-                "sha256:965e4a05ef364e7b973dd17fc765f42233415974d773e82144c9bbaaaea5d089",
-                "sha256:96e88745a55b88a7c64fa49bceff363a1a27d9a64e04019c2281049444a571e3",
-                "sha256:99eb6cafb6ba90e436684e08dad8be1637efb71c4f2180ee6b8f940739406e78",
-                "sha256:9adf58f5d64e474bed00d69bcd86ec4bcaa4123bfa70a65ce72e424bfb88ed96",
-                "sha256:9b1af95c3a967bf1da94f253e56b6286b50af23392a886720f563c547e48e964",
-                "sha256:a0aa9417994d91301056f3d0038af1199eb7adc86e646a36b9e050b06f526597",
-                "sha256:a0f9bb6c80e6efcde93ffc51256d5cfb2155ff8f78292f074f60f9e70b942d99",
-                "sha256:a127ae76092974abfbfa38ca2d12cbeddcdeac0fb71f9627cc1135bedaf9d51a",
-                "sha256:aaf305d6d40bd9632198c766fb64f0c1a83ca5b667f16c1e79e1661ab5060140",
-                "sha256:aca1c196f407ec7cf04dcbb15d19a43c507a81f7ffc45b690899d6a76ac9fda7",
-                "sha256:ace6ca218308447b9077c14ea4ef381ba0b67ee78d64046b3f19cf4e1139ad16",
-                "sha256:b416f03d37d27290cb93597335a2f85ed446731200705b22bb927405320de903",
-                "sha256:bf548479d336726d7a0eceb6e767e179fbde37833ae42794602631a070d630f1",
-                "sha256:c1170d6b195555644f0616fd6ed929dfcf6333b8675fcca044ae5ab110ded296",
-                "sha256:c380b27d041209b849ed246b111b7c166ba36d7933ec6e41175fd15ab9eb1572",
-                "sha256:c446d2245ba29820d405315083d55299a796695d747efceb5717a8b450324115",
-                "sha256:c830a02caeb789633863b466b9de10c015bded434deb3ec87c768e53752ad22a",
-                "sha256:cb841572862f629b99725ebaec3287fc6d275be9b14443ea746c1dd325053cbd",
-                "sha256:cfa4561277f677ecf651e2b22dc43e8f5368b74a25a8f7d1d4a3a243e573f2d4",
-                "sha256:cfcc2c53c06f2ccb8976fb5c71d448bdd0a07d26d8e07e321c103416444c7ad1",
-                "sha256:d3c6b54e304c60c4181da1c9dadf83e4a54fd266a99c70ba646a9baa626819eb",
-                "sha256:d3d403753c9d5adc04d4694d35cf0391f0f3d57c8e0030aac09d7678fa8030aa",
-                "sha256:d9c206c29b46cfd343ea7cdfe1232443072bbb270d6a46f59c259460db76779a",
-                "sha256:e49eb4e95ff6fd7c0c402508894b1ef0e01b99a44320ba7d8ecbabefddcc5569",
-                "sha256:f8286396b351785801a976b1e85ea88e937712ee2c3ac653710a4a57a8da5d9c",
-                "sha256:f8fc330c3370a81bbf3f88557097d1ea26cd8b019d6433aa59f71195f5ddebbf",
-                "sha256:fbd359831c1657d69bb81f0db962905ee05e5e9451913b18b831febfe0519082",
-                "sha256:fe7e1c262d3392afcf5071df9afa574544f28eac825284596ac6db56e6d11062",
-                "sha256:fed1e1cf6a42577953abbe8e6cf2fe2f566daebde7c34724ec8803c4c0cda579"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==9.5.0"
-        },
-        "pipreqs": {
-            "hashes": [
-                "sha256:a17f167880b6921be37533ce4c81ddc6e22b465c107aad557db43b1add56a99b",
-                "sha256:e522b9ed54aa3e8b7978ff251ab7a9af2f75d2cd8de4c102e881b666a79a308e"
+                "sha256:00e65f5e822decd501e374b0650146063fbb30a7264b4d2744bdd7b913e0cab5",
+                "sha256:040586f7d37b34547153fa383f7f9aed68b738992380ac911447bb78f2abe530",
+                "sha256:0b6eb5502f45a60a3f411c63187db83a3d3107887ad0d036c13ce836f8a36f1d",
+                "sha256:1f62406a884ae75fb2f818694469519fb685cc7eaff05d3451a9ebe55c646891",
+                "sha256:22c10cc517668d44b211717fd9775799ccec4124b9a7f7b3635fc5386e584992",
+                "sha256:3400aae60685b06bb96f99a21e1ada7bc7a413d5f49bce739828ecd9391bb8f7",
+                "sha256:349930d6e9c685c089284b013478d6f76e3a534e36ddfa912cde493f235372f3",
+                "sha256:368ab3dfb5f49e312231b6f27b8820c823652b7cd29cfbd34090565a015e99ba",
+                "sha256:38250a349b6b390ee6047a62c086d3817ac69022c127f8a5dc058c31ccef17f3",
+                "sha256:3a684105f7c32488f7153905a4e3015a3b6c7182e106fe3c37fbb5ef3e6994c3",
+                "sha256:3a82c40d706d9aa9734289740ce26460a11aeec2d9c79b7af87bb35f0073c12f",
+                "sha256:3b08d4cc24f471b2c8ca24ec060abf4bebc6b144cb89cba638c720546b1cf538",
+                "sha256:3ed64f9ca2f0a95411e88a4efbd7a29e5ce2cea36072c53dd9d26d9c76f753b3",
+                "sha256:3f07ea8d2f827d7d2a49ecf1639ec02d75ffd1b88dcc5b3a61bbb37a8759ad8d",
+                "sha256:520f2a520dc040512699f20fa1c363eed506e94248d71f85412b625026f6142c",
+                "sha256:5c6e3df6bdd396749bafd45314871b3d0af81ff935b2d188385e970052091017",
+                "sha256:608bfdee0d57cf297d32bcbb3c728dc1da0907519d1784962c5f0c68bb93e5a3",
+                "sha256:685ac03cc4ed5ebc15ad5c23bc555d68a87777586d970c2c3e216619a5476223",
+                "sha256:76de421f9c326da8f43d690110f0e79fe3ad1e54be811545d7d91898b4c8493e",
+                "sha256:76edb0a1fa2b4745fb0c99fb9fb98f8b180a1bbceb8be49b087e0b21867e77d3",
+                "sha256:7be600823e4c8631b74e4a0d38384c73f680e6105a7d3c6824fcf226c178c7e6",
+                "sha256:81ff539a12457809666fef6624684c008e00ff6bf455b4b89fd00a140eecd640",
+                "sha256:88af2003543cc40c80f6fca01411892ec52b11021b3dc22ec3bc9d5afd1c5334",
+                "sha256:8c11160913e3dd06c8ffdb5f233a4f254cb449f4dfc0f8f4549eda9e542c93d1",
+                "sha256:8f8182b523b2289f7c415f589118228d30ac8c355baa2f3194ced084dac2dbba",
+                "sha256:9211e7ad69d7c9401cfc0e23d49b69ca65ddd898976d660a2fa5904e3d7a9baa",
+                "sha256:92be919bbc9f7d09f7ae343c38f5bb21c973d2576c1d45600fce4b74bafa7ac0",
+                "sha256:9c82b5b3e043c7af0d95792d0d20ccf68f61a1fec6b3530e718b688422727396",
+                "sha256:9f7c16705f44e0504a3a2a14197c1f0b32a95731d251777dcb060aa83022cb2d",
+                "sha256:9fb218c8a12e51d7ead2a7c9e101a04982237d4855716af2e9499306728fb485",
+                "sha256:a74ba0c356aaa3bb8e3eb79606a87669e7ec6444be352870623025d75a14a2bf",
+                "sha256:b4f69b3700201b80bb82c3a97d5e9254084f6dd5fb5b16fc1a7b974260f89f43",
+                "sha256:c189af0545965fa8d3b9613cfdb0cd37f9d71349e0f7750e1fd704648d475ed2",
+                "sha256:c1fbe7621c167ecaa38ad29643d77a9ce7311583761abf7836e1510c580bf3dd",
+                "sha256:c7cf14a27b0d6adfaebb3ae4153f1e516df54e47e42dcc073d7b3d76111a8d86",
+                "sha256:c9f72a021fbb792ce98306ffb0c348b3c9cb967dce0f12a49aa4c3d3fdefa967",
+                "sha256:cd25d2a9d2b36fcb318882481367956d2cf91329f6892fe5d385c346c0649629",
+                "sha256:ce543ed15570eedbb85df19b0a1a7314a9c8141a36ce089c0a894adbfccb4568",
+                "sha256:ce7b031a6fc11365970e6a5686d7ba8c63e4c1cf1ea143811acbb524295eabed",
+                "sha256:d35e3c8d9b1268cbf5d3670285feb3528f6680420eafe35cccc686b73c1e330f",
+                "sha256:d50b6aec14bc737742ca96e85d6d0a5f9bfbded018264b3b70ff9d8c33485551",
+                "sha256:d5d0dae4cfd56969d23d94dc8e89fb6a217be461c69090768227beb8ed28c0a3",
+                "sha256:d5db32e2a6ccbb3d34d87c87b432959e0db29755727afb37290e10f6e8e62614",
+                "sha256:d72e2ecc68a942e8cf9739619b7f408cc7b272b279b56b2c83c6123fcfa5cdff",
+                "sha256:d737a602fbd82afd892ca746392401b634e278cb65d55c4b7a8f48e9ef8d008d",
+                "sha256:d80cf684b541685fccdd84c485b31ce73fc5c9b5d7523bf1394ce134a60c6883",
+                "sha256:db24668940f82321e746773a4bc617bfac06ec831e5c88b643f91f122a785684",
+                "sha256:dbc02381779d412145331789b40cc7b11fdf449e5d94f6bc0b080db0a56ea3f0",
+                "sha256:dffe31a7f47b603318c609f378ebcd57f1554a3a6a8effbc59c3c69f804296de",
+                "sha256:edf4392b77bdc81f36e92d3a07a5cd072f90253197f4a52a55a8cec48a12483b",
+                "sha256:efe8c0681042536e0d06c11f48cebe759707c9e9abf880ee213541c5b46c5bf3",
+                "sha256:f31f9fdbfecb042d046f9d91270a0ba28368a723302786c0009ee9b9f1f60199",
+                "sha256:f88a0b92277de8e3ca715a0d79d68dc82807457dae3ab8699c758f07c20b3c51",
+                "sha256:faaf07ea35355b01a35cb442dd950d8f1bb5b040a7787791a535de13db15ed90"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==0.4.13"
+            "markers": "python_version >= '3.8'",
+            "version": "==10.0.0"
         },
         "portalocker": {
             "hashes": [
@@ -1311,6 +1368,13 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==5.9.5"
         },
+        "py-cpuinfo": {
+            "hashes": [
+                "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690",
+                "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5"
+            ],
+            "version": "==9.0.0"
+        },
         "pyarrow": {
             "hashes": [
                 "sha256:03a10daad957970e914920b793f6a49416699e791f4c827927fd4e4d892a5d16",
@@ -1347,6 +1411,48 @@
             "index": "pypi",
             "version": "==8.0.0"
         },
+        "pydantic": {
+            "hashes": [
+                "sha256:008c5e266c8aada206d0627a011504e14268a62091450210eda7c07fabe6963e",
+                "sha256:0588788a9a85f3e5e9ebca14211a496409cb3deca5b6971ff37c556d581854e7",
+                "sha256:08a6c32e1c3809fbc49debb96bf833164f3438b3696abf0fbeceb417d123e6eb",
+                "sha256:16928fdc9cb273c6af00d9d5045434c39afba5f42325fb990add2c241402d151",
+                "sha256:174899023337b9fc685ac8adaa7b047050616136ccd30e9070627c1aaab53a13",
+                "sha256:192c608ad002a748e4a0bed2ddbcd98f9b56df50a7c24d9a931a8c5dd053bd3d",
+                "sha256:1954f8778489a04b245a1e7b8b22a9d3ea8ef49337285693cf6959e4b757535e",
+                "sha256:2417de68290434461a266271fc57274a138510dca19982336639484c73a07af6",
+                "sha256:265a60da42f9f27e0b1014eab8acd3e53bd0bad5c5b4884e98a55f8f596b2c19",
+                "sha256:331c031ba1554b974c98679bd0780d89670d6fd6f53f5d70b10bdc9addee1713",
+                "sha256:373c0840f5c2b5b1ccadd9286782852b901055998136287828731868027a724f",
+                "sha256:3f34739a89260dfa420aa3cbd069fbcc794b25bbe5c0a214f8fb29e363484b66",
+                "sha256:41e0bb6efe86281623abbeeb0be64eab740c865388ee934cd3e6a358784aca6e",
+                "sha256:4400015f15c9b464c9db2d5d951b6a780102cfa5870f2c036d37c23b56f7fc1b",
+                "sha256:44e51ba599c3ef227e168424e220cd3e544288c57829520dc90ea9cb190c3248",
+                "sha256:469adf96c8e2c2bbfa655fc7735a2a82f4c543d9fee97bd113a7fb509bf5e622",
+                "sha256:5b02d24f7b2b365fed586ed73582c20f353a4c50e4be9ba2c57ab96f8091ddae",
+                "sha256:7522a7666157aa22b812ce14c827574ddccc94f361237ca6ea8bb0d5c38f1629",
+                "sha256:787cf23e5a0cde753f2eabac1b2e73ae3844eb873fd1f5bdbff3048d8dbb7604",
+                "sha256:8268a735a14c308923e8958363e3a3404f6834bb98c11f5ab43251a4e410170c",
+                "sha256:8dc77064471780262b6a68fe67e013298d130414d5aaf9b562c33987dbd2cf4f",
+                "sha256:a451ccab49971af043ec4e0d207cbc8cbe53dbf148ef9f19599024076fe9c25b",
+                "sha256:a6c098d4ab5e2d5b3984d3cb2527e2d6099d3de85630c8934efcfdc348a9760e",
+                "sha256:abade85268cc92dff86d6effcd917893130f0ff516f3d637f50dadc22ae93999",
+                "sha256:bc64eab9b19cd794a380179ac0e6752335e9555d214cfcb755820333c0784cb3",
+                "sha256:c3339a46bbe6013ef7bdd2844679bfe500347ac5742cd4019a88312aa58a9847",
+                "sha256:d185819a7a059550ecb85d5134e7d40f2565f3dd94cfd870132c5f91a89cf58c",
+                "sha256:d7781f1d13b19700b7949c5a639c764a077cbbdd4322ed505b449d3ca8edcb36",
+                "sha256:e297897eb4bebde985f72a46a7552a7556a3dd11e7f76acda0c1093e3dbcf216",
+                "sha256:e6cbfbd010b14c8a905a7b10f9fe090068d1744d46f9e0c021db28daeb8b6de1",
+                "sha256:e9738b0f2e6c70f44ee0de53f2089d6002b10c33264abee07bdb5c7f03038303",
+                "sha256:e9baf78b31da2dc3d3f346ef18e58ec5f12f5aaa17ac517e2ffd026a92a87588",
+                "sha256:ef55392ec4bb5721f4ded1096241e4b7151ba6d50a50a80a2526c854f42e6a2f",
+                "sha256:f66d479cf7eb331372c470614be6511eae96f1f120344c25f3f9bb59fb1b5528",
+                "sha256:fe429898f2c9dd209bd0632a606bddc06f8bce081bbd03d1c775a45886e2c1cb",
+                "sha256:ff44c5e89315b15ff1f7fdaf9853770b810936d6b01a7bcecaa227d2f8fe444f"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.10.11"
+        },
         "pyjwt": {
             "hashes": [
                 "sha256:ba2b425b15ad5ef12f200dc67dd56af4e26de2331f965c5439994dad075876e1",
@@ -1355,39 +1461,6 @@
             "markers": "python_version >= '3.7'",
             "version": "==2.7.0"
         },
-        "pyrsistent": {
-            "hashes": [
-                "sha256:016ad1afadf318eb7911baa24b049909f7f3bb2c5b1ed7b6a8f21db21ea3faa8",
-                "sha256:1a2994773706bbb4995c31a97bc94f1418314923bd1048c6d964837040376440",
-                "sha256:20460ac0ea439a3e79caa1dbd560344b64ed75e85d8703943e0b66c2a6150e4a",
-                "sha256:3311cb4237a341aa52ab8448c27e3a9931e2ee09561ad150ba94e4cfd3fc888c",
-                "sha256:3a8cb235fa6d3fd7aae6a4f1429bbb1fec1577d978098da1252f0489937786f3",
-                "sha256:3ab2204234c0ecd8b9368dbd6a53e83c3d4f3cab10ecaf6d0e772f456c442393",
-                "sha256:42ac0b2f44607eb92ae88609eda931a4f0dfa03038c44c772e07f43e738bcac9",
-                "sha256:49c32f216c17148695ca0e02a5c521e28a4ee6c5089f97e34fe24163113722da",
-                "sha256:4b774f9288dda8d425adb6544e5903f1fb6c273ab3128a355c6b972b7df39dcf",
-                "sha256:4c18264cb84b5e68e7085a43723f9e4c1fd1d935ab240ce02c0324a8e01ccb64",
-                "sha256:5a474fb80f5e0d6c9394d8db0fc19e90fa540b82ee52dba7d246a7791712f74a",
-                "sha256:64220c429e42a7150f4bfd280f6f4bb2850f95956bde93c6fda1b70507af6ef3",
-                "sha256:878433581fc23e906d947a6814336eee031a00e6defba224234169ae3d3d6a98",
-                "sha256:99abb85579e2165bd8522f0c0138864da97847875ecbd45f3e7e2af569bfc6f2",
-                "sha256:a2471f3f8693101975b1ff85ffd19bb7ca7dd7c38f8a81701f67d6b4f97b87d8",
-                "sha256:aeda827381f5e5d65cced3024126529ddc4289d944f75e090572c77ceb19adbf",
-                "sha256:b735e538f74ec31378f5a1e3886a26d2ca6351106b4dfde376a26fc32a044edc",
-                "sha256:c147257a92374fde8498491f53ffa8f4822cd70c0d85037e09028e478cababb7",
-                "sha256:c4db1bd596fefd66b296a3d5d943c94f4fac5bcd13e99bffe2ba6a759d959a28",
-                "sha256:c74bed51f9b41c48366a286395c67f4e894374306b197e62810e0fdaf2364da2",
-                "sha256:c9bb60a40a0ab9aba40a59f68214eed5a29c6274c83b2cc206a359c4a89fa41b",
-                "sha256:cc5d149f31706762c1f8bda2e8c4f8fead6e80312e3692619a75301d3dbb819a",
-                "sha256:ccf0d6bd208f8111179f0c26fdf84ed7c3891982f2edaeae7422575f47e66b64",
-                "sha256:e42296a09e83028b3476f7073fcb69ffebac0e66dbbfd1bd847d61f74db30f19",
-                "sha256:e8f2b814a3dc6225964fa03d8582c6e0b6650d68a232df41e3cc1b66a5d2f8d1",
-                "sha256:f0774bf48631f3a20471dd7c5989657b639fd2d285b861237ea9e82c36a415a9",
-                "sha256:f0e7c4b2f77593871e918be000b96c8107da48444d57005b6a6bc61fb4331b2c"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==0.19.3"
-        },
         "python-dateutil": {
             "hashes": [
                 "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
@@ -1475,6 +1548,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==4.0.5"
         },
+        "referencing": {
+            "hashes": [
+                "sha256:90cb53782d550ba28d2166ef3f55731f38397def8832baac5d45235f1995e35e",
+                "sha256:d3c8f323ee1480095da44d55917cfb8278d73d6b4d5f677e3e40eb21314ac67f"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.29.1"
+        },
         "regex": {
             "hashes": [
                 "sha256:0385e73da22363778ef2324950e08b689abdf0b108a7d8decb403ad7f5191938",
@@ -1607,6 +1688,87 @@
             ],
             "version": "==1.3.8"
         },
+        "rpds-py": {
+            "hashes": [
+                "sha256:0c62fdb01111da948e8446caaefebf2ca4307a58fcbc10039b48d0db7205397c",
+                "sha256:0df2ae878fd99342415a42659f3bcee34b12441a509033e0ab04c6e301895607",
+                "sha256:0dfec14a64759186153115d987f9e60b1fa7e8e14a00004a02482e696199e554",
+                "sha256:0e8da63b9baa154ec9ddd6dd397893830d17e5812ceb50edbae8122d8ecb9f2e",
+                "sha256:13aed64f2e0bef04a0eae6d1d1295f901f6c1640d1e20264dc3b19d62ef1721d",
+                "sha256:180963bb3e1fcc6ed6313ece5e065f0df4021a7eb7016084d3cbc90cd2a8af3e",
+                "sha256:1891903e567d728175c0475a1f0ffc1d1580013b0b265b9e2f1b8c93d58b2d05",
+                "sha256:18a97bb7f211b247346983092186927c517153ac155c611f43ca83d5ee93a3e2",
+                "sha256:1f60a7eb96fedcc5bf59761d33ac0f2f127d75f8c4b99ed0f138fc0f3601c537",
+                "sha256:22afed13c6ad4ccdc650ad44cbc06f835f4f30de201bd4ee2afc09bde06a357a",
+                "sha256:246a410e540ce7f635c6ad1b7aa00b7dcfc966c5f97217e41092c3f764dac4bf",
+                "sha256:247a1fd9bcdb373380cb8c0041417c9995fc163c9d88a39f5ec3d960114aca22",
+                "sha256:25a4b357ba7540d7cc709915699d67d505c8177cdb30824127092e1b4886b504",
+                "sha256:2d6a26953d3b291dd7e79e43bb203ed134ca889e63c8ebbc65e3ff98154303ef",
+                "sha256:300b8579740b06e246238b730e636f314a7d8dc475be1868650f5d3ddc29a0d8",
+                "sha256:316a66b0379a9e954872593aa2eb6d61949da2ecc8f572b4dafb07aa0e247171",
+                "sha256:358c7a976482537d26fcbdc7808e7cc25f64816fe89681f9aa8bef13e16c7370",
+                "sha256:36266e2e49b36ec6cc148e36667030d8e7f1c009edd2ca978ab09ed9c5a7589a",
+                "sha256:3660bd2afb23e1ca685df0d3092208fe6c7b2cae8c0be199b3da6d1a4bc91dc5",
+                "sha256:42eb3030665ee7a5c03fd4db6b8db1983aa91bcdffbed0f4687751deb2a94a7c",
+                "sha256:4333b5c2801e44bf342207a7700378a1013f300116c9553ce1ffbc73047d2a02",
+                "sha256:4b5164fdad37847e90683d3748dca7c4604f47ecd21e2d651dafc80393d1d314",
+                "sha256:4c66a5f9ca9c5fcb2527ad969541521f7216db713a7bd63aee52c3f5efa5924a",
+                "sha256:5b7c87503843a036f898d44c47326a117d23b6269a9f1574adf83d7bb051b839",
+                "sha256:657348b35a4c2e7c2340bf0bc37597900037bd87e9db7e6282711aaa77256e16",
+                "sha256:681ef7a21e6990583533c3a3038b7176f5e51e5d345fe2d9109b54f6bffcabcd",
+                "sha256:6b1b235b890373f785507f8f6a644a4f191b7195939e7f6108dc0e5e4fab57fd",
+                "sha256:6e85ea159f2d2132d4fcb71abb7df9683314f6073bf8ee9f9bd576440245e59c",
+                "sha256:6ee396f63a1f540fb3aecb8cc698180d30573a661be47fb3fff45cbd2b5d4686",
+                "sha256:7110854662ccf8db84b90e4624301ef5311cafff7e5f2a63f2d7cc0fc1a75b60",
+                "sha256:755a837fb7053dbf511fba26343423bd80d3b75a5d7f57f6e407c2fe5ae46682",
+                "sha256:7628b2080538faa4a1243b0002678cae7111af68ae7b5aa6cd8526762cace868",
+                "sha256:78c5577f99d2edc9eed9ec39fae27b73d04d1b2462aff6f6b11207e0364fc40d",
+                "sha256:79ac819182795a2168ed11075c7362de368f360244fb7cea8274c222b2d55365",
+                "sha256:7bd8dbc1c63668124e5e48e601d32f1053cfd5a86004ae0e55dc9ba8b1e7de29",
+                "sha256:87d74c2526115daa9d805a66377997602185a837ff7ecceed9d27e625c383572",
+                "sha256:8ac1e47ee4cb2dbd714537e63c67086eec63f56b13208fe450ae5be4f3d65671",
+                "sha256:8e1251d6690f356a305192089017da83999cded1d7e2405660d14c1dff976af7",
+                "sha256:8f64ef5700ff8fded62b12d1ea55463031cc5a353b670ed7146126c6cbf28788",
+                "sha256:961828c668140796c4963edb14cd968666d5414b9b5829997a4f475fd917906e",
+                "sha256:9af40bb89e40932e04c0cc1fb516249d6b3ae68ceebd984fdc592a6673244e50",
+                "sha256:9b331fa451d725258c1ad0ae6816cf36d55294e5cb68338cf91550b9a448a48f",
+                "sha256:a747477838a90b9264f434b5399309f9badb32c80ff3e9c4f6d5b87fddcbaa09",
+                "sha256:ac596301c7723809ecb6ec4cb2564b2cadc06f8c07b6348a56fcbf1ae043d751",
+                "sha256:b189640d59afa8aeff59865fa9d314bca97987c378950f215297e15d64ae1124",
+                "sha256:b3ed0d3498b69159db0d5db1393c8bae4df51cf936b2ef5cda53d800acab0019",
+                "sha256:b5a55fd01f61df19e4f53fd67b63ca9bf47559632c3f7d4037faa06d3a6fed54",
+                "sha256:b5cbcbd6451a4af2048fc0b21c15981462f6a378cb039aa53612c6e39958064e",
+                "sha256:b949e86affe17c8828d82936c51d7aa9b686511f9ac99a4b1de596d7140c8083",
+                "sha256:bfe98ad05400e7ac7a1df791645db08c66adc92d7a6449a406111303a265b71a",
+                "sha256:c0fe018430e5e8e1d8b513bcbccdb0ea34b9fd81c32b3a49c41a109fade17cfb",
+                "sha256:c26b1a0425c038cc23cf5770a47d7a7382fe68d6d10fb2a52c2896ca94e72550",
+                "sha256:c7019b2522af6b835118177b6b53f7ed08da28061aa5d44e06286be09799e7a4",
+                "sha256:c97e4cbe722474d17c309ee49e09331ceffe345bb72e2b10c6c740788871b122",
+                "sha256:cafbfa8f3a27e592bdcc420497e0c9a957c9f812b6c6ebfb7f961409215ba82d",
+                "sha256:cea42c2f52e37877e6b877bce64d109a6b0213e32545ecc70d4492d2a4641b8f",
+                "sha256:d2f8666fde7cfd9cdbc6c223876b39697d387f0215d12ed25147e9efec2dff5a",
+                "sha256:d418d2fd8c0fffe2897bc3e15a2e2ec87abf29076f0c36898cc33fb7881c2cbe",
+                "sha256:d4334405f6c73c29ff94521f78ad53ebb76a9c1b8dafea75852f9f64c3679cf7",
+                "sha256:d6696c2a002e982e89975063939a2579623d6d6b24634b147848ec9f35dad7d6",
+                "sha256:d7e46f52272ceecc42c05ad869b068b2dbfb6eb5643bcccecd2327d3cded5a2e",
+                "sha256:db71c665fc7ddb9ac53d7b69dc588493c0b71635b28fc8ff64b31eb9db5b3461",
+                "sha256:ddb5226b11f4fce98c6e47c07819fbbfdda6d3fd529cd176ad8a1e0b98a70b05",
+                "sha256:df3097abf5fd09bfcd8f6fd02d052b25cc3e160b3ee71b6fbd831b2cd516c958",
+                "sha256:e086610118163400a1822af0ee857581c0e047aa50a9c3543d17fbe65183a339",
+                "sha256:e1700fba17dd63c9c7d5cb03cf838db23cf938dd5cdb1eecb6ba8b8da62c3e73",
+                "sha256:e261fa453ad50fe1d9287fa21d962cdbcd3d495cf1160dd0f893883040c455b6",
+                "sha256:e4e11f71673905d9e8735b8dd4ef8fa85a82e6003851fe46f9bdc51aebc2cd5d",
+                "sha256:ea8c4a1232c7bacf73366b0490dda7f67d25958aec1c2f099b6753c8bfe84427",
+                "sha256:ed823997c3300b541da0fcc9eee8fbe6740e07939ffa432824cfca287472d652",
+                "sha256:ee42ce4ef46ea334ce8ab63d5a57c7fd78238c9c7293b3caa6dfedf11bd28773",
+                "sha256:f1c84912d77b01651488bbe392df593b4c5852e213477e268ebbb7c799059d78",
+                "sha256:f81a570a20f9fce617d728f4e3bdc05bfbb68afa2e795ec0c52544a7923de517",
+                "sha256:f8a1cc37e2395002381510a6862c29634acd67edfe5774661a6c48d8617acae7",
+                "sha256:fffd98c1fd6b38df35e471549c2486d826af0fda6ca55c0bbbb956b291e7f2ae"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.8.8"
+        },
         "s3transfer": {
             "hashes": [
                 "sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971",
@@ -2205,11 +2367,11 @@
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:5d8c9dac95c27d20df12fb1d97b9793ab8b2af8a3a525e68c80e21060c161771",
-                "sha256:935ccf31549830cda708b42289d44b6f74084d616a00be651601a4f968e77c82"
+                "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36",
+                "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.7.0"
+            "version": "==4.7.1"
         },
         "uri-template": {
             "hashes": [
@@ -2248,6 +2410,13 @@
             ],
             "version": "==1.13"
         },
+        "webencodings": {
+            "hashes": [
+                "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78",
+                "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"
+            ],
+            "version": "==0.5.1"
+        },
         "websocket-client": {
             "hashes": [
                 "sha256:c951af98631d24f8df89ab1019fc365f2227c0892f12fd150e935607c79dd0dd",
@@ -2360,13 +2529,6 @@
             "markers": "python_version >= '3.6'",
             "version": "==3.2.0"
         },
-        "yarg": {
-            "hashes": [
-                "sha256:4f9cebdc00fac946c9bf2783d634e538a71c7d280a4d806d45fd4dc0ef441492",
-                "sha256:55695bf4d1e3e7f756496c36a69ba32c40d18f821e38f61d028f6049e5e15911"
-            ],
-            "version": "==0.1.9"
-        },
         "yarl": {
             "hashes": [
                 "sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571",
@@ -2490,11 +2652,11 @@
         },
         "click": {
             "hashes": [
-                "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e",
-                "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"
+                "sha256:2739815aaa5d2c986a88f1e9230c55e17f0caad3d958a5e13ad0797c166db9e3",
+                "sha256:b97d0c74955da062a7d4ef92fadb583806a585b2ea81958a81bd72726cbb8e37"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.1.3"
+            "version": "==8.1.4"
         },
         "coverage": {
             "hashes": [
@@ -2659,11 +2821,11 @@
         },
         "platformdirs": {
             "hashes": [
-                "sha256:b0cabcb11063d21a0b261d557acb0a9d2126350e63b70cdf7db6347baea456dc",
-                "sha256:ca9ed98ce73076ba72e092b23d3c93ea6c4e186b3f1c3dad6edd98ff6ffcca2e"
+                "sha256:cec7b889196b9144d088e4c57d9ceef7374f6c39694ad1577a0aab50d27ea28c",
+                "sha256:f87ca4fcff7d2b0f81c6a748a77973d7af0f4d526f98f308477c3c436c74d528"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.8.0"
+            "version": "==3.8.1"
         },
         "pluggy": {
             "hashes": [
@@ -2761,11 +2923,11 @@
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:5d8c9dac95c27d20df12fb1d97b9793ab8b2af8a3a525e68c80e21060c161771",
-                "sha256:935ccf31549830cda708b42289d44b6f74084d616a00be651601a4f968e77c82"
+                "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36",
+                "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.7.0"
+            "version": "==4.7.1"
         },
         "wheel": {
             "hashes": [
diff --git a/app_utils/utils.py b/app_utils/utils.py
index 67e7338528..1aeef79c9e 100644
--- a/app_utils/utils.py
+++ b/app_utils/utils.py
@@ -136,21 +136,41 @@ def start_process(
     #     )
     else:
         free_port = find_free_port()
-        p = subprocess.Popen(
-            [
-                "env",
-                f"CUDA_VISIBLE_DEVICES={','.join(gpu_list)}",
-                "torchrun",
-                f"--nproc_per_node={str(num_gpus)}",
-                f"--master_port={str(free_port)}",
-                "train_wave.py",
-                "-Y",
-                config_name,
-                "-Q",
-                ",".join([str(x) for x in process_queue]),
-            ],
-            env=env,
-        )
+        if cfg.environment.use_deepspeed:
+            logger.info("Starting deepspeed...")
+            p = subprocess.Popen(
+                [
+                    "env",
+                    "deepspeed",
+                    "--include",
+                    f"localhost:{','.join(gpu_list)}",
+                    "--master_port",
+                    f"{str(free_port)}",
+                    "train_wave.py",
+                    "-Y",
+                    config_name,
+                    "-Q",
+                    ",".join([str(x) for x in process_queue]),
+                ],
+                env=env,
+            )
+        else:
+            logger.info("Starting torchrun...")
+            p = subprocess.Popen(
+                [
+                    "env",
+                    f"CUDA_VISIBLE_DEVICES={','.join(gpu_list)}",
+                    "torchrun",
+                    f"--nproc_per_node={str(num_gpus)}",
+                    f"--master_port={str(free_port)}",
+                    "train_wave.py",
+                    "-Y",
+                    config_name,
+                    "-Q",
+                    ",".join([str(x) for x in process_queue]),
+                ],
+                env=env,
+            )
     logger.info(f"Percentage of RAM memory used: {psutil.virtual_memory().percent}")
 
     return p
diff --git a/llm_studio/python_configs/text_causal_language_modeling_config.py b/llm_studio/python_configs/text_causal_language_modeling_config.py
index 4af3733332..d981b326bb 100644
--- a/llm_studio/python_configs/text_causal_language_modeling_config.py
+++ b/llm_studio/python_configs/text_causal_language_modeling_config.py
@@ -391,6 +391,7 @@ class ConfigNLPCausalLMEnvironment(DefaultConfig):
 
     compile_model: bool = False
     use_fsdp: bool = False
+    use_deepspeed: bool = False
 
     find_unused_parameters: bool = False
     trust_remote_code: bool = True
diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index 87c204acd8..4deb87ff57 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -7,6 +7,7 @@
 from typing import Any, Dict
 
 import coolname
+import deepspeed
 import numpy as np
 import torch
 from torch.cuda.amp import autocast
@@ -169,6 +170,62 @@ def load_checkpoint(
         logger.info(f"Weights loaded from: {weights_path}")
 
 
+def deepspeed_initialize(
+    model: torch.nn.Module,
+    optimizer: torch.optim.Optimizer,
+    lr_scheduler: torch.optim.lr_scheduler._LRScheduler,
+    training_data: torch.utils.data.Dataset,
+    validating_data: torch.utils.data.Dataset,
+    cfg: Any,
+):
+    mconfig = AutoConfig.from_pretrained(cfg.llm_backbone)
+    if hasattr(mconfig, "hidden_size"):
+        model_hidden_size = mconfig.hidden_size
+    elif hasattr(mconfig, "d_model"):
+        model_hidden_size = mconfig.d_model
+    else:
+        raise Exception(f"deepspeed do not support {cfg.llm_backbone}")
+    ds_config = {
+        "fp16": {
+            "enabled": True if cfg.architecture.backbone_dtype == "float16" else False
+        },
+        "bf16": {
+            "enabled": True if cfg.architecture.backbone_dtype == "bfloat16" else False
+        },
+        # https://www.deepspeed.ai/docs/config-json/#zero-optimizations-for-fp16-training
+        "zero_optimization": {
+            "stage": 3,
+            "offload_param": {"device": "cpu", "pin_memory": True},
+            "overlap_comm": True,
+            "contiguous_gradients": True,
+            "reduce_bucket_size": model_hidden_size * model_hidden_size,
+            "stage3_prefetch_bucket_size": 0.9 * model_hidden_size * model_hidden_size,
+            "stage3_param_persistence_threshold": 10 * model_hidden_size,
+        },
+        "steps_per_print": 2000,
+        "train_batch_size": cfg.training.batch_size * cfg.environment._world_size,
+        "train_micro_batch_size_per_gpu": 1,
+        "wall_clock_breakdown": False,
+    }
+
+    # from transformers.deepspeed import HfDeepSpeedConfig
+    #     dschf = HfDeepSpeedConfig(ds_config)  # keep this object alive
+
+    model, optimizer, train_dataloader, scheduler = deepspeed.initialize(
+        model=model,
+        optimizer=optimizer,
+        lr_scheduler=lr_scheduler,
+        training_data=training_data,
+        config_params=ds_config,
+    )
+    _, _, val_dataloader, _ = deepspeed.initialize(
+        model=torch.nn.Linear(1, 1),
+        training_data=validating_data,
+        config_params=ds_config,
+    )
+    return model, optimizer, train_dataloader, val_dataloader, scheduler
+
+
 def wrap_model_distributed(model: torch.nn.Module, cfg: Any, fsdp: bool):
     if fsdp:
         auto_wrap_policy = None
@@ -214,59 +271,66 @@ def get_optimizer(model: torch.nn.Module, cfg: Any) -> torch.optim.Optimizer:
     Returns:
         Optimizer
     """
-
-    no_decay = ["bias", "LayerNorm.weight"]
-    differential_layers = cfg.training.differential_learning_rate_layers
-    optimizer = Optimizers.get(cfg.training.optimizer)(
-        [
-            {
-                "params": [
-                    param
-                    for name, param in model.named_parameters()
-                    if (not any(layer in name for layer in differential_layers))
-                    and (not any(nd in name for nd in no_decay))
-                    # and param.requires_grad
-                ],
-                "lr": cfg.training.learning_rate,
-                "weight_decay": cfg.training.weight_decay,
-            },
-            {
-                "params": [
-                    param
-                    for name, param in model.named_parameters()
-                    if (not any(layer in name for layer in differential_layers))
-                    and (any(nd in name for nd in no_decay))
-                    # and param.requires_grad
-                ],
-                "lr": cfg.training.learning_rate,
-                "weight_decay": 0,
-            },
-            {
-                "params": [
-                    param
-                    for name, param in model.named_parameters()
-                    if (any(layer in name for layer in differential_layers))
-                    and (not any(nd in name for nd in no_decay))
-                    # and param.requires_grad
-                ],
-                "lr": cfg.training.differential_learning_rate,
-                "weight_decay": cfg.training.weight_decay,
-            },
-            {
-                "params": [
-                    param
-                    for name, param in model.named_parameters()
-                    if (any(layer in name for layer in differential_layers))
-                    and (any(nd in name for nd in no_decay))
-                    # and param.requires_grad
-                ],
-                "lr": cfg.training.differential_learning_rate,
-                "weight_decay": 0,
-            },
-        ],
-        lr=cfg.training.learning_rate,
-        weight_decay=cfg.training.weight_decay,
-    )
+    if cfg.environment.use_deepspeed and cfg.training.lora:
+        logger.info(
+            "Deepspeed /w Lora training do not support differential learning rate."
+        )
+        optimizer = Optimizers.get(cfg.training.optimizer)(
+            model.parameters(), lr=cfg.training.learning_rate
+        )
+    else:
+        no_decay = ["bias", "LayerNorm.weight"]
+        differential_layers = cfg.training.differential_learning_rate_layers
+        optimizer = Optimizers.get(cfg.training.optimizer)(
+            [
+                {
+                    "params": [
+                        param
+                        for name, param in model.named_parameters()
+                        if (not any(layer in name for layer in differential_layers))
+                        and (not any(nd in name for nd in no_decay))
+                        # and param.requires_grad
+                    ],
+                    "lr": cfg.training.learning_rate,
+                    "weight_decay": cfg.training.weight_decay,
+                },
+                {
+                    "params": [
+                        param
+                        for name, param in model.named_parameters()
+                        if (not any(layer in name for layer in differential_layers))
+                        and (any(nd in name for nd in no_decay))
+                        # and param.requires_grad
+                    ],
+                    "lr": cfg.training.learning_rate,
+                    "weight_decay": 0,
+                },
+                {
+                    "params": [
+                        param
+                        for name, param in model.named_parameters()
+                        if (any(layer in name for layer in differential_layers))
+                        and (not any(nd in name for nd in no_decay))
+                        # and param.requires_grad
+                    ],
+                    "lr": cfg.training.differential_learning_rate,
+                    "weight_decay": cfg.training.weight_decay,
+                },
+                {
+                    "params": [
+                        param
+                        for name, param in model.named_parameters()
+                        if (any(layer in name for layer in differential_layers))
+                        and (any(nd in name for nd in no_decay))
+                        # and param.requires_grad
+                    ],
+                    "lr": cfg.training.differential_learning_rate,
+                    "weight_decay": 0,
+                },
+            ],
+            lr=cfg.training.learning_rate,
+            weight_decay=cfg.training.weight_decay,
+        )
 
     return optimizer
 
diff --git a/train.py b/train.py
index 5a57f97f79..a2a908c070 100644
--- a/train.py
+++ b/train.py
@@ -15,6 +15,7 @@
 from distutils import util
 from typing import Any, Callable, Dict, Tuple
 
+import deepspeed
 import numpy as np
 import pandas as pd
 import torch
@@ -50,6 +51,7 @@
 )
 from llm_studio.src.utils.modeling_utils import (
     check_disk_space,
+    deepspeed_initialize,
     get_number_of_validation_epochs,
     get_optimizer,
     get_scheduler,
@@ -178,6 +180,22 @@ def run_train(
     optimizer = get_optimizer(model=model, cfg=cfg)
     scheduler = get_scheduler(cfg=cfg, optimizer=optimizer, epoch_steps=epoch_steps)
 
+    if cfg.environment.use_deepspeed:
+        (
+            model,
+            optimizer,
+            train_dataloader,
+            val_dataloader,
+            scheduler,
+        ) = deepspeed_initialize(
+            model=model,
+            optimizer=optimizer,
+            lr_scheduler=scheduler,
+            training_data=train_dataloader.dataset,
+            validating_data=val_dataloader.dataset,
+            cfg=cfg,
+        )
+
     scaler: GradScaler | ShardedGradScaler | None = None
     if cfg.environment.mixed_precision:
         if cfg.environment.use_fsdp:
@@ -229,8 +247,10 @@ def run_train(
         if cfg.environment._local_rank == 0:
             logger.info(f"Training Epoch: {epoch + 1} / {cfg.training.epochs}")
 
-        if cfg.environment._distributed and hasattr(
-            train_dataloader.sampler, "set_epoch"
+        if (
+            cfg.environment._distributed
+            and not cfg.environment.use_deepspeed
+            and hasattr(train_dataloader.sampler, "set_epoch")
         ):
             train_dataloader.sampler.set_epoch(epoch)  # type: ignore
 
@@ -356,7 +376,10 @@ def run_train(
                     loss = loss / cfg.training.grad_accumulation
 
                 # Backward pass
-                if cfg.environment.mixed_precision:
+                if (
+                    cfg.environment.mixed_precision
+                    and not cfg.environment.use_deepspeed
+                ):
                     scaler.scale(loss).backward()  # type: ignore
                     if itr % cfg.training.grad_accumulation == 0:
                         if cfg.training.gradient_clip > 0:
@@ -368,7 +391,10 @@ def run_train(
                         scaler.update()
                         optimizer.zero_grad(set_to_none=True)
                 else:
-                    loss.backward()
+                    if cfg.environment.use_deepspeed:
+                        model.backward(loss)
+                    else:
+                        loss.backward()
                     if itr % cfg.training.grad_accumulation == 0:
                         if cfg.training.gradient_clip > 0:
                             torch.nn.utils.clip_grad_norm_(
@@ -489,6 +515,10 @@ def run(cfg: Any) -> None:
         cfg: config object with all the hyperparameters
     """
 
+    if cfg.environment.use_deepspeed:
+        # To avoid warnings about parallelism in tokenizers
+        os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
     os.makedirs(cfg.output_directory, exist_ok=True)
 
     # Force evaluation if user trains 0 epochs
@@ -512,7 +542,10 @@ def run(cfg: Any) -> None:
     if cfg.environment._distributed:
         cfg.environment._local_rank = int(os.environ["LOCAL_RANK"])
         cfg.environment._device = "cuda:%d" % cfg.environment._local_rank
-        torch.distributed.init_process_group(backend="nccl", init_method="env://")
+        if cfg.environment.use_deepspeed:
+            deepspeed.init_distributed()
+        else:
+            torch.distributed.init_process_group(backend="nccl", init_method="env://")
         cfg.environment._cpu_comm = torch.distributed.new_group(backend="gloo")
 
         cfg.environment._world_size = torch.distributed.get_world_size()
@@ -625,10 +658,11 @@ def run(cfg: Any) -> None:
                     param.requires_grad = True
                     param.data = param.data.float()
 
-    if cfg.environment._distributed:
+    if cfg.environment._distributed and not cfg.environment.use_deepspeed:
         model = wrap_model_distributed(model, cfg, cfg.environment.use_fsdp)
 
-    if cfg.environment.compile_model:
+    # deepspeed do not support torch.compile
+    if cfg.environment.compile_model and not cfg.environment.use_deepspeed:
         if cfg.environment._distributed:
             model.module.backbone = torch.compile(model.module.backbone)
         else:

From 5ef4792a0d86ea2d0fbaf43252238d35a2f6cd93 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Wed, 19 Jul 2023 05:53:01 -0700
Subject: [PATCH 02/52] shard

---
 llm_studio/src/utils/modeling_utils.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index 4deb87ff57..bb0039d0de 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -193,9 +193,12 @@ def deepspeed_initialize(
             "enabled": True if cfg.architecture.backbone_dtype == "bfloat16" else False
         },
         # https://www.deepspeed.ai/docs/config-json/#zero-optimizations-for-fp16-training
+        "zero_force_ds_cpu_optimizer": False,
         "zero_optimization": {
             "stage": 3,
-            "offload_param": {"device": "cpu", "pin_memory": True},
+            "mics_shard_size": cfg.environment._world_size,
+#             "offload_optimizer": {"device": "cpu"},
+#             "offload_param": {"device": "cpu", "pin_memory": True},
             "overlap_comm": True,
             "contiguous_gradients": True,
             "reduce_bucket_size": model_hidden_size * model_hidden_size,

From b5fac57b3d61fcd5c2b6a50cd672134a090ea80d Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Tue, 25 Jul 2023 01:20:49 -0700
Subject: [PATCH 03/52] full param deepspeed works by this commit

---
 llm_studio/src/utils/modeling_utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index bb0039d0de..51191b4e62 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -199,11 +199,13 @@ def deepspeed_initialize(
             "mics_shard_size": cfg.environment._world_size,
 #             "offload_optimizer": {"device": "cpu"},
 #             "offload_param": {"device": "cpu", "pin_memory": True},
-            "overlap_comm": True,
+            "overlap_comm": False,
             "contiguous_gradients": True,
             "reduce_bucket_size": model_hidden_size * model_hidden_size,
             "stage3_prefetch_bucket_size": 0.9 * model_hidden_size * model_hidden_size,
             "stage3_param_persistence_threshold": 10 * model_hidden_size,
+            "stage3_max_live_parameters": 1e9,
+            "stage3_max_reuse_distance": 1e9,
         },
         "steps_per_print": 2000,
         "train_batch_size": cfg.training.batch_size * cfg.environment._world_size,

From 0f7086b66fb21c73edaf0b80d061cf10c9cb58c2 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Wed, 26 Jul 2023 00:08:47 -0700
Subject: [PATCH 04/52] offload optimizer & documentation

---
 .../experiments/_deepspeed-offload-optimizer.mdx    |  1 +
 .../_deepspeed-stage3-max-live-parameters.mdx       |  1 +
 .../_deepspeed-stage3-max-reuse-distance.mdx        |  1 +
 .../docs/tooltips/experiments/_use-deepspeed.mdx    |  1 +
 .../text_causal_language_modeling_config.py         | 10 ++++++++++
 llm_studio/src/utils/modeling_utils.py              | 13 ++++++-------
 6 files changed, 20 insertions(+), 7 deletions(-)
 create mode 100644 documentation/docs/tooltips/experiments/_deepspeed-offload-optimizer.mdx
 create mode 100644 documentation/docs/tooltips/experiments/_deepspeed-stage3-max-live-parameters.mdx
 create mode 100644 documentation/docs/tooltips/experiments/_deepspeed-stage3-max-reuse-distance.mdx
 create mode 100644 documentation/docs/tooltips/experiments/_use-deepspeed.mdx

diff --git a/documentation/docs/tooltips/experiments/_deepspeed-offload-optimizer.mdx b/documentation/docs/tooltips/experiments/_deepspeed-offload-optimizer.mdx
new file mode 100644
index 0000000000..ffd3b28c50
--- /dev/null
+++ b/documentation/docs/tooltips/experiments/_deepspeed-offload-optimizer.mdx
@@ -0,0 +1 @@
+Whether to offload optimizer to cpu for saving more GPU ram during training. Note that turn on offload_optimizer would further make training speed slower.
\ No newline at end of file
diff --git a/documentation/docs/tooltips/experiments/_deepspeed-stage3-max-live-parameters.mdx b/documentation/docs/tooltips/experiments/_deepspeed-stage3-max-live-parameters.mdx
new file mode 100644
index 0000000000..3c2bafea9c
--- /dev/null
+++ b/documentation/docs/tooltips/experiments/_deepspeed-stage3-max-live-parameters.mdx
@@ -0,0 +1 @@
+The maximum number of parameters resident per GPU before releasing. Smaller values use less memory, but make training speed slower.
\ No newline at end of file
diff --git a/documentation/docs/tooltips/experiments/_deepspeed-stage3-max-reuse-distance.mdx b/documentation/docs/tooltips/experiments/_deepspeed-stage3-max-reuse-distance.mdx
new file mode 100644
index 0000000000..268b542995
--- /dev/null
+++ b/documentation/docs/tooltips/experiments/_deepspeed-stage3-max-reuse-distance.mdx
@@ -0,0 +1 @@
+Do not release a parameter if it will be reused within this threshold of parameters. Smaller values use less memory, but make training speed slower.
\ No newline at end of file
diff --git a/documentation/docs/tooltips/experiments/_use-deepspeed.mdx b/documentation/docs/tooltips/experiments/_use-deepspeed.mdx
new file mode 100644
index 0000000000..884e2ba04e
--- /dev/null
+++ b/documentation/docs/tooltips/experiments/_use-deepspeed.mdx
@@ -0,0 +1 @@
+Whether to use deepspeed for saving GPU ram during training. Note that turn on deepspeed would make training speed slower.
\ No newline at end of file
diff --git a/llm_studio/python_configs/text_causal_language_modeling_config.py b/llm_studio/python_configs/text_causal_language_modeling_config.py
index d981b326bb..0a1969a949 100644
--- a/llm_studio/python_configs/text_causal_language_modeling_config.py
+++ b/llm_studio/python_configs/text_causal_language_modeling_config.py
@@ -392,6 +392,10 @@ class ConfigNLPCausalLMEnvironment(DefaultConfig):
     compile_model: bool = False
     use_fsdp: bool = False
     use_deepspeed: bool = False
+    deepspeed_offload_optimizer: bool = False
+    deepspeed_stage3_max_live_parameters: int = 1e9
+    deepspeed_stage3_max_reuse_distance: int = 1e9
+
 
     find_unused_parameters: bool = False
     trust_remote_code: bool = True
@@ -421,6 +425,12 @@ def __post_init__(self):
 
         self._possible_values["number_of_workers"] = (1, multiprocessing.cpu_count(), 1)
         self._possible_values["seed"] = possible_values.Number(step=1, min=-1)
+        self._possible_values["deepspeed_stage3_max_live_parameters"] = possible_values.Number(step=1, min=1e7)
+        self._possible_values["deepspeed_stage3_max_reuse_distance"] = possible_values.Number(step=1, min=1e7)
+        self._nesting.add(
+            ["deepspeed_offload_optimizer", "deepspeed_stage3_max_live_parameters", "deepspeed_stage3_max_reuse_distance"],
+            [Dependency(key="use_deepspeed", value=False, is_set=False)],
+        )
 
 
 @dataclass
diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index 51191b4e62..1309a68f63 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -197,24 +197,23 @@ def deepspeed_initialize(
         "zero_optimization": {
             "stage": 3,
             "mics_shard_size": cfg.environment._world_size,
-#             "offload_optimizer": {"device": "cpu"},
-#             "offload_param": {"device": "cpu", "pin_memory": True},
             "overlap_comm": False,
             "contiguous_gradients": True,
             "reduce_bucket_size": model_hidden_size * model_hidden_size,
             "stage3_prefetch_bucket_size": 0.9 * model_hidden_size * model_hidden_size,
             "stage3_param_persistence_threshold": 10 * model_hidden_size,
-            "stage3_max_live_parameters": 1e9,
-            "stage3_max_reuse_distance": 1e9,
+            "stage3_max_live_parameters": cfg.environment.deepspeed_stage3_max_live_parameters,
+            "stage3_max_reuse_distance": cfg.environment.deepspeed_stage3_max_reuse_distance,
         },
         "steps_per_print": 2000,
         "train_batch_size": cfg.training.batch_size * cfg.environment._world_size,
         "train_micro_batch_size_per_gpu": 1,
         "wall_clock_breakdown": False,
     }
-
-    # from transformers.deepspeed import HfDeepSpeedConfig
-    #     dschf = HfDeepSpeedConfig(ds_config)  # keep this object alive
+    if cfg.environment.deepspeed_offload_optimizer:
+        ds_config["zero_optimization"]["offload_optimizer"] = {"device": "cpu", "pin_memory": True}
+#     if cfg.environment.deepspeed_offload_param:  ### RuntimeError: Tensors must be CUDA and dense
+#         ds_config["zero_optimization"]["offload_param"] = {"device": "cpu", "pin_memory": True}
 
     model, optimizer, train_dataloader, scheduler = deepspeed.initialize(
         model=model,

From 687c4560e75112220aa4dc7fdeb3b54b1e62ef54 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Wed, 2 Aug 2023 08:05:02 -0700
Subject: [PATCH 05/52] format & fix save deepspeed weight

---
 .../text_causal_language_modeling_config.py   | 15 +++--
 llm_studio/src/utils/modeling_utils.py        | 31 +++++++---
 train.py                                      | 60 ++++++++++++++++---
 3 files changed, 85 insertions(+), 21 deletions(-)

diff --git a/llm_studio/python_configs/text_causal_language_modeling_config.py b/llm_studio/python_configs/text_causal_language_modeling_config.py
index 0a1969a949..b98b36f152 100644
--- a/llm_studio/python_configs/text_causal_language_modeling_config.py
+++ b/llm_studio/python_configs/text_causal_language_modeling_config.py
@@ -396,7 +396,6 @@ class ConfigNLPCausalLMEnvironment(DefaultConfig):
     deepspeed_stage3_max_live_parameters: int = 1e9
     deepspeed_stage3_max_reuse_distance: int = 1e9
 
-
     find_unused_parameters: bool = False
     trust_remote_code: bool = True
     huggingface_branch: str = "main"
@@ -425,10 +424,18 @@ def __post_init__(self):
 
         self._possible_values["number_of_workers"] = (1, multiprocessing.cpu_count(), 1)
         self._possible_values["seed"] = possible_values.Number(step=1, min=-1)
-        self._possible_values["deepspeed_stage3_max_live_parameters"] = possible_values.Number(step=1, min=1e7)
-        self._possible_values["deepspeed_stage3_max_reuse_distance"] = possible_values.Number(step=1, min=1e7)
+        self._possible_values[
+            "deepspeed_stage3_max_live_parameters"
+        ] = possible_values.Number(step=1, min=1e7)
+        self._possible_values[
+            "deepspeed_stage3_max_reuse_distance"
+        ] = possible_values.Number(step=1, min=1e7)
         self._nesting.add(
-            ["deepspeed_offload_optimizer", "deepspeed_stage3_max_live_parameters", "deepspeed_stage3_max_reuse_distance"],
+            [
+                "deepspeed_offload_optimizer",
+                "deepspeed_stage3_max_live_parameters",
+                "deepspeed_stage3_max_reuse_distance",
+            ],
             [Dependency(key="use_deepspeed", value=False, is_set=False)],
         )
 
diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index 1309a68f63..d3b91dca27 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -187,23 +187,33 @@ def deepspeed_initialize(
         raise Exception(f"deepspeed do not support {cfg.llm_backbone}")
     ds_config = {
         "fp16": {
-            "enabled": True if cfg.architecture.backbone_dtype == "float16" else False
+            "enabled": True if cfg.architecture.backbone_dtype == "float16" else False,
+            "loss_scale_window": 100,
         },
         "bf16": {
-            "enabled": True if cfg.architecture.backbone_dtype == "bfloat16" else False
+            "enabled": True if cfg.architecture.backbone_dtype == "bfloat16" else False,
+            "loss_scale_window": 100,
         },
         # https://www.deepspeed.ai/docs/config-json/#zero-optimizations-for-fp16-training
         "zero_force_ds_cpu_optimizer": False,
         "zero_optimization": {
             "stage": 3,
-            "mics_shard_size": cfg.environment._world_size,
             "overlap_comm": False,
             "contiguous_gradients": True,
             "reduce_bucket_size": model_hidden_size * model_hidden_size,
+            # zero3
+            "mics_shard_size": cfg.environment._world_size,
             "stage3_prefetch_bucket_size": 0.9 * model_hidden_size * model_hidden_size,
             "stage3_param_persistence_threshold": 10 * model_hidden_size,
-            "stage3_max_live_parameters": cfg.environment.deepspeed_stage3_max_live_parameters,
-            "stage3_max_reuse_distance": cfg.environment.deepspeed_stage3_max_reuse_distance,
+            "stage3_max_live_parameters":
+                cfg.environment.deepspeed_stage3_max_live_parameters,
+            "stage3_max_reuse_distance":
+                cfg.environment.deepspeed_stage3_max_reuse_distance,
+            # zero++
+            # "reduce_scatter": True,
+            # "zero_quantized_weights": True,
+            # "zero_hpz_partition_size": 16,
+            # "zero_quantized_gradients": True,
         },
         "steps_per_print": 2000,
         "train_batch_size": cfg.training.batch_size * cfg.environment._world_size,
@@ -211,9 +221,14 @@ def deepspeed_initialize(
         "wall_clock_breakdown": False,
     }
     if cfg.environment.deepspeed_offload_optimizer:
-        ds_config["zero_optimization"]["offload_optimizer"] = {"device": "cpu", "pin_memory": True}
-#     if cfg.environment.deepspeed_offload_param:  ### RuntimeError: Tensors must be CUDA and dense
-#         ds_config["zero_optimization"]["offload_param"] = {"device": "cpu", "pin_memory": True}
+        ds_config["zero_optimization"]["offload_optimizer"] = {
+            "device": "cpu",
+            "pin_memory": True,
+        }
+    # TODO: RuntimeError: Tensors must be CUDA and dense
+    # if cfg.environment.deepspeed_offload_param:
+    #     ds_config["zero_optimization"]["offload_param"] =
+    #         {"device": "cpu", "pin_memory": True}
 
     model, optimizer, train_dataloader, scheduler = deepspeed.initialize(
         model=model,
diff --git a/train.py b/train.py
index a2a908c070..424c1110a2 100644
--- a/train.py
+++ b/train.py
@@ -10,6 +10,7 @@
 import argparse
 import gc
 import logging
+import shutil
 import sys
 import time
 from distutils import util
@@ -19,6 +20,7 @@
 import numpy as np
 import pandas as pd
 import torch
+from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
 from torch.cuda.amp import GradScaler, autocast
 from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler
 from torch.utils.data import DataLoader
@@ -468,25 +470,65 @@ def run_train(
                 val_loss, val_metric = run_eval(
                     cfg=cfg, model=model, val_dataloader=val_dataloader, val_df=val_df
                 )
-                if cfg.environment._local_rank == 0:
-                    if cfg.training.save_best_checkpoint:
-                        if objective_op(val_metric, best_val_metric):
-                            checkpoint_path = cfg.output_directory
+
+                if cfg.training.save_best_checkpoint:
+                    if objective_op(val_metric, best_val_metric):
+                        checkpoint_path = cfg.output_directory
+                        if cfg.environment._local_rank == 0:
                             logger.info(
                                 f"Saving best model checkpoint: "
                                 f"val_{cfg.prediction.metric} {best_val_metric:.5} -> "
                                 f"{val_metric:.5} to {checkpoint_path}"
                             )
-                            save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
-                            best_val_metric = val_metric
-                    else:
-                        checkpoint_path = cfg.output_directory
+                        if cfg.environment.use_deepspeed:
+                            model.save_checkpoint(
+                                os.path.join(checkpoint_path, "ds_checkpoint")
+                            )
+                            if cfg.environment._local_rank == 0:
+                                logger.info("ds checkpoint saved")
+                                state_dict = get_fp32_state_dict_from_zero_checkpoint(
+                                    os.path.join(checkpoint_path, "ds_checkpoint")
+                                )
+                                torch.save(
+                                    {"model": state_dict},
+                                    os.path.join(checkpoint_path, "checkpoint.pth"),
+                                )
+                                shutil.rmtree(
+                                    os.path.join(checkpoint_path, "ds_checkpoint")
+                                )
+                        else:
+                            if cfg.environment._local_rank == 0:
+                                save_checkpoint(
+                                    model=model, path=checkpoint_path, cfg=cfg
+                                )
+                        best_val_metric = val_metric
+                else:
+                    checkpoint_path = cfg.output_directory
+                    if cfg.environment._local_rank == 0:
                         logger.info(
                             f"Saving last model checkpoint: "
                             f"val_loss {val_loss:.5}, val_{cfg.prediction.metric} "
                             f"{val_metric:.5} to {checkpoint_path}"
                         )
-                        save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
+                    if cfg.environment.use_deepspeed:
+                        model.save_checkpoint(
+                            os.path.join(checkpoint_path, "ds_checkpoint")
+                        )
+                        if cfg.environment._local_rank == 0:
+                            logger.info("ds checkpoint saved")
+                            state_dict = get_fp32_state_dict_from_zero_checkpoint(
+                                os.path.join(checkpoint_path, "ds_checkpoint")
+                            )
+                            torch.save(
+                                {"model": state_dict},
+                                os.path.join(checkpoint_path, "checkpoint.pth"),
+                            )
+                            shutil.rmtree(
+                                os.path.join(checkpoint_path, "ds_checkpoint")
+                            )
+                    else:
+                        if cfg.environment._local_rank == 0:
+                            save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
 
                 model.train()
 

From 3b7ff0dc13f854a9ca021fe34db613621c88d38f Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Wed, 2 Aug 2023 21:40:30 -0700
Subject: [PATCH 06/52] format & update save_checkpoint

---
 llm_studio/src/utils/modeling_utils.py | 37 ++++++++++++++--------
 train.py                               | 43 ++------------------------
 2 files changed, 26 insertions(+), 54 deletions(-)

diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index d3b91dca27..a900fd71d4 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -10,6 +10,7 @@
 import deepspeed
 import numpy as np
 import torch
+from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
 from torch.cuda.amp import autocast
 from torch.distributed.fsdp.fully_sharded_data_parallel import (
     FullyShardedDataParallel,
@@ -76,15 +77,27 @@ def save_checkpoint(model: torch.nn.Module, path: str, cfg: Any):
         Dictionary with all the keys to save
     """
 
-    model = unwrap_model(model)
-
-    if hasattr(cfg.training, "lora") and cfg.training.lora:
-        model.backbone.save_pretrained(path)
-
-    checkpoint = {"model": model.state_dict()}
-
-    if path is not None:
-        torch.save(checkpoint, os.path.join(path, "checkpoint.pth"))
+    if cfg.environment.use_deepspeed:
+        if path is not None:
+            # gather model params from all ranks
+            model.save_checkpoint(os.path.join(path, "ds_checkpoint"))
+            if cfg.environment._local_rank == 0:
+                # load to cpu
+                state_dict = get_fp32_state_dict_from_zero_checkpoint(
+                    os.path.join(path, "ds_checkpoint")
+                )
+                # save as normal checkpoint that can be loaded by `load_state_dict`
+                checkpoint = {"model": state_dict}
+                torch.save(checkpoint, os.path.join(path, "checkpoint.pth"))
+                shutil.rmtree(os.path.join(path, "ds_checkpoint"))
+    else:
+        if cfg.environment._local_rank == 0:
+            model = unwrap_model(model)
+            if hasattr(cfg.training, "lora") and cfg.training.lora:
+                model.backbone.save_pretrained(path)
+            checkpoint = {"model": model.state_dict()}
+            if path is not None:
+                torch.save(checkpoint, os.path.join(path, "checkpoint.pth"))
 
 
 def load_model_weights(
@@ -205,10 +218,8 @@ def deepspeed_initialize(
             "mics_shard_size": cfg.environment._world_size,
             "stage3_prefetch_bucket_size": 0.9 * model_hidden_size * model_hidden_size,
             "stage3_param_persistence_threshold": 10 * model_hidden_size,
-            "stage3_max_live_parameters":
-                cfg.environment.deepspeed_stage3_max_live_parameters,
-            "stage3_max_reuse_distance":
-                cfg.environment.deepspeed_stage3_max_reuse_distance,
+            "stage3_max_live_parameters": cfg.environment.deepspeed_stage3_max_live_parameters,  # noqa: E501
+            "stage3_max_reuse_distance": cfg.environment.deepspeed_stage3_max_reuse_distance,  # noqa: E501
             # zero++
             # "reduce_scatter": True,
             # "zero_quantized_weights": True,
diff --git a/train.py b/train.py
index 424c1110a2..6fb57d8fd9 100644
--- a/train.py
+++ b/train.py
@@ -20,7 +20,6 @@
 import numpy as np
 import pandas as pd
 import torch
-from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
 from torch.cuda.amp import GradScaler, autocast
 from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler
 from torch.utils.data import DataLoader
@@ -480,27 +479,7 @@ def run_train(
                                 f"val_{cfg.prediction.metric} {best_val_metric:.5} -> "
                                 f"{val_metric:.5} to {checkpoint_path}"
                             )
-                        if cfg.environment.use_deepspeed:
-                            model.save_checkpoint(
-                                os.path.join(checkpoint_path, "ds_checkpoint")
-                            )
-                            if cfg.environment._local_rank == 0:
-                                logger.info("ds checkpoint saved")
-                                state_dict = get_fp32_state_dict_from_zero_checkpoint(
-                                    os.path.join(checkpoint_path, "ds_checkpoint")
-                                )
-                                torch.save(
-                                    {"model": state_dict},
-                                    os.path.join(checkpoint_path, "checkpoint.pth"),
-                                )
-                                shutil.rmtree(
-                                    os.path.join(checkpoint_path, "ds_checkpoint")
-                                )
-                        else:
-                            if cfg.environment._local_rank == 0:
-                                save_checkpoint(
-                                    model=model, path=checkpoint_path, cfg=cfg
-                                )
+                        save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
                         best_val_metric = val_metric
                 else:
                     checkpoint_path = cfg.output_directory
@@ -510,25 +489,7 @@ def run_train(
                             f"val_loss {val_loss:.5}, val_{cfg.prediction.metric} "
                             f"{val_metric:.5} to {checkpoint_path}"
                         )
-                    if cfg.environment.use_deepspeed:
-                        model.save_checkpoint(
-                            os.path.join(checkpoint_path, "ds_checkpoint")
-                        )
-                        if cfg.environment._local_rank == 0:
-                            logger.info("ds checkpoint saved")
-                            state_dict = get_fp32_state_dict_from_zero_checkpoint(
-                                os.path.join(checkpoint_path, "ds_checkpoint")
-                            )
-                            torch.save(
-                                {"model": state_dict},
-                                os.path.join(checkpoint_path, "checkpoint.pth"),
-                            )
-                            shutil.rmtree(
-                                os.path.join(checkpoint_path, "ds_checkpoint")
-                            )
-                    else:
-                        if cfg.environment._local_rank == 0:
-                            save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
+                    save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
 
                 model.train()
 

From 105a849edac3e738b89fea51e2fd51184a2ed045 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Fri, 4 Aug 2023 01:12:54 -0700
Subject: [PATCH 07/52] update pipfile

---
 Pipfile      |   21 +-
 Pipfile.lock | 1358 ++++++++++++++++++++++++++------------------------
 2 files changed, 710 insertions(+), 669 deletions(-)

diff --git a/Pipfile b/Pipfile
index eaefd3158a..e71fe91a4e 100644
--- a/Pipfile
+++ b/Pipfile
@@ -5,18 +5,18 @@ verify_ssl = true
 
 [[source]]
 name = "pytorch"
-url = "https://download.pytorch.org/whl/cu117"
+url = "https://download.pytorch.org/whl/cu118"
 verify_ssl = false
 
 [requires]
 python_version = "3.10"
 
 [packages]
-torch = {index = "pytorch", version = "==2.0.0+cu117"}
-torchvision = {index = "pytorch", version = "==0.15.1+cu117"}
-torchaudio = {index = "pytorch", version = "==2.0.1+cu117"}
+torch = {index = "pytorch", version = "==2.0.0+cu118"}
+torchvision = {index = "pytorch", version = "==0.15.1+cu118"}
+torchaudio = {index = "pytorch", version = "==2.0.1+cu118"}
 tqdm = ">=4.65.0, <5.0.0"
-transformers = "==4.30.1"
+transformers = "==4.31.0"
 numpy = ">=1.23.2, <2.0.0"
 pandas = ">=1.4.2, <2.0.0"
 scikit-learn = "==1.0.2"
@@ -38,18 +38,19 @@ fastparquet = "==0.8.1"
 gputil = "==1.4.0"
 huggingface-hub = "==0.14.1"
 starlette= ">=0.27.0, <0.28.0"
-peft = {ref = "main", git = "https://github.com/huggingface/peft"}
 bitsandbytes = "==0.39.1"
 accelerate = "==0.20.3"
 openai = "==0.27.7"
 einops = "==0.6.1"
 datasets = "==2.11.0"
-neptune = ">=1.2.0, <2.0.0"
+neptune = "==1.3.1"
 Jinja2 = ">=3.1.2, <4.0.0"
 tenacity = ">=8.2.2, <9.0.0"
 h2o-wave = "0.26"
-tiktoken = "*"
-deepspeed = ">=0.9.2"
+tiktoken = "0.4.0"
+hf-transfer = "0.1.3"
+peft = "0.4.0"
+deepspeed = ">=0.10.0"
 mpi4py = ">=3.1.4"
 
 [dev-packages]
@@ -66,4 +67,4 @@ pytest-dependency = "==0.5.1"
 types-requests = "==2.28.11.17"
 types-toml = "==0.10.8.6"
 types-pyyaml = "==6.0.12.9"
-mypy = "==1.2.0"
+mypy = "==1.2.0"
\ No newline at end of file
diff --git a/Pipfile.lock b/Pipfile.lock
index 5bd56dc5a7..9daae5bc60 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "196b33e3d862a1c80dd953eefcdd066965c4a8b03017a41a6a66b3c09ea35ba2"
+            "sha256": "18b059325e2f2d74b0582877d64ab84d3c85613753bb1e4724162c9f3c0e093c"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -15,7 +15,7 @@
             },
             {
                 "name": "pytorch",
-                "url": "https://download.pytorch.org/whl/cu117",
+                "url": "https://download.pytorch.org/whl/cu118",
                 "verify_ssl": false
             }
         ]
@@ -31,96 +31,96 @@
         },
         "aiohttp": {
             "hashes": [
-                "sha256:03543dcf98a6619254b409be2d22b51f21ec66272be4ebda7b04e6412e4b2e14",
-                "sha256:03baa76b730e4e15a45f81dfe29a8d910314143414e528737f8589ec60cf7391",
-                "sha256:0a63f03189a6fa7c900226e3ef5ba4d3bd047e18f445e69adbd65af433add5a2",
-                "sha256:10c8cefcff98fd9168cdd86c4da8b84baaa90bf2da2269c6161984e6737bf23e",
-                "sha256:147ae376f14b55f4f3c2b118b95be50a369b89b38a971e80a17c3fd623f280c9",
-                "sha256:176a64b24c0935869d5bbc4c96e82f89f643bcdf08ec947701b9dbb3c956b7dd",
-                "sha256:17b79c2963db82086229012cff93ea55196ed31f6493bb1ccd2c62f1724324e4",
-                "sha256:1a45865451439eb320784918617ba54b7a377e3501fb70402ab84d38c2cd891b",
-                "sha256:1b3ea7edd2d24538959c1c1abf97c744d879d4e541d38305f9bd7d9b10c9ec41",
-                "sha256:22f6eab15b6db242499a16de87939a342f5a950ad0abaf1532038e2ce7d31567",
-                "sha256:3032dcb1c35bc330134a5b8a5d4f68c1a87252dfc6e1262c65a7e30e62298275",
-                "sha256:33587f26dcee66efb2fff3c177547bd0449ab7edf1b73a7f5dea1e38609a0c54",
-                "sha256:34ce9f93a4a68d1272d26030655dd1b58ff727b3ed2a33d80ec433561b03d67a",
-                "sha256:3a80464982d41b1fbfe3154e440ba4904b71c1a53e9cd584098cd41efdb188ef",
-                "sha256:3b90467ebc3d9fa5b0f9b6489dfb2c304a1db7b9946fa92aa76a831b9d587e99",
-                "sha256:3d89efa095ca7d442a6d0cbc755f9e08190ba40069b235c9886a8763b03785da",
-                "sha256:3d8ef1a630519a26d6760bc695842579cb09e373c5f227a21b67dc3eb16cfea4",
-                "sha256:3f43255086fe25e36fd5ed8f2ee47477408a73ef00e804cb2b5cba4bf2ac7f5e",
-                "sha256:40653609b3bf50611356e6b6554e3a331f6879fa7116f3959b20e3528783e699",
-                "sha256:41a86a69bb63bb2fc3dc9ad5ea9f10f1c9c8e282b471931be0268ddd09430b04",
-                "sha256:493f5bc2f8307286b7799c6d899d388bbaa7dfa6c4caf4f97ef7521b9cb13719",
-                "sha256:4a6cadebe132e90cefa77e45f2d2f1a4b2ce5c6b1bfc1656c1ddafcfe4ba8131",
-                "sha256:4c745b109057e7e5f1848c689ee4fb3a016c8d4d92da52b312f8a509f83aa05e",
-                "sha256:4d347a172f866cd1d93126d9b239fcbe682acb39b48ee0873c73c933dd23bd0f",
-                "sha256:4dac314662f4e2aa5009977b652d9b8db7121b46c38f2073bfeed9f4049732cd",
-                "sha256:4ddaae3f3d32fc2cb4c53fab020b69a05c8ab1f02e0e59665c6f7a0d3a5be54f",
-                "sha256:5393fb786a9e23e4799fec788e7e735de18052f83682ce2dfcabaf1c00c2c08e",
-                "sha256:59f029a5f6e2d679296db7bee982bb3d20c088e52a2977e3175faf31d6fb75d1",
-                "sha256:5a7bdf9e57126dc345b683c3632e8ba317c31d2a41acd5800c10640387d193ed",
-                "sha256:5b3f2e06a512e94722886c0827bee9807c86a9f698fac6b3aee841fab49bbfb4",
-                "sha256:5ce45967538fb747370308d3145aa68a074bdecb4f3a300869590f725ced69c1",
-                "sha256:5e14f25765a578a0a634d5f0cd1e2c3f53964553a00347998dfdf96b8137f777",
-                "sha256:618c901dd3aad4ace71dfa0f5e82e88b46ef57e3239fc7027773cb6d4ed53531",
-                "sha256:652b1bff4f15f6287550b4670546a2947f2a4575b6c6dff7760eafb22eacbf0b",
-                "sha256:6c08e8ed6fa3d477e501ec9db169bfac8140e830aa372d77e4a43084d8dd91ab",
-                "sha256:6ddb2a2026c3f6a68c3998a6c47ab6795e4127315d2e35a09997da21865757f8",
-                "sha256:6e601588f2b502c93c30cd5a45bfc665faaf37bbe835b7cfd461753068232074",
-                "sha256:6e74dd54f7239fcffe07913ff8b964e28b712f09846e20de78676ce2a3dc0bfc",
-                "sha256:7235604476a76ef249bd64cb8274ed24ccf6995c4a8b51a237005ee7a57e8643",
-                "sha256:7ab43061a0c81198d88f39aaf90dae9a7744620978f7ef3e3708339b8ed2ef01",
-                "sha256:7c7837fe8037e96b6dd5cfcf47263c1620a9d332a87ec06a6ca4564e56bd0f36",
-                "sha256:80575ba9377c5171407a06d0196b2310b679dc752d02a1fcaa2bc20b235dbf24",
-                "sha256:80a37fe8f7c1e6ce8f2d9c411676e4bc633a8462844e38f46156d07a7d401654",
-                "sha256:8189c56eb0ddbb95bfadb8f60ea1b22fcfa659396ea36f6adcc521213cd7b44d",
-                "sha256:854f422ac44af92bfe172d8e73229c270dc09b96535e8a548f99c84f82dde241",
-                "sha256:880e15bb6dad90549b43f796b391cfffd7af373f4646784795e20d92606b7a51",
-                "sha256:8b631e26df63e52f7cce0cce6507b7a7f1bc9b0c501fcde69742130b32e8782f",
-                "sha256:8c29c77cc57e40f84acef9bfb904373a4e89a4e8b74e71aa8075c021ec9078c2",
-                "sha256:91f6d540163f90bbaef9387e65f18f73ffd7c79f5225ac3d3f61df7b0d01ad15",
-                "sha256:92c0cea74a2a81c4c76b62ea1cac163ecb20fb3ba3a75c909b9fa71b4ad493cf",
-                "sha256:9bcb89336efa095ea21b30f9e686763f2be4478f1b0a616969551982c4ee4c3b",
-                "sha256:a1f4689c9a1462f3df0a1f7e797791cd6b124ddbee2b570d34e7f38ade0e2c71",
-                "sha256:a3fec6a4cb5551721cdd70473eb009d90935b4063acc5f40905d40ecfea23e05",
-                "sha256:a5d794d1ae64e7753e405ba58e08fcfa73e3fad93ef9b7e31112ef3c9a0efb52",
-                "sha256:a86d42d7cba1cec432d47ab13b6637bee393a10f664c425ea7b305d1301ca1a3",
-                "sha256:adfbc22e87365a6e564c804c58fc44ff7727deea782d175c33602737b7feadb6",
-                "sha256:aeb29c84bb53a84b1a81c6c09d24cf33bb8432cc5c39979021cc0f98c1292a1a",
-                "sha256:aede4df4eeb926c8fa70de46c340a1bc2c6079e1c40ccf7b0eae1313ffd33519",
-                "sha256:b744c33b6f14ca26b7544e8d8aadff6b765a80ad6164fb1a430bbadd593dfb1a",
-                "sha256:b7a00a9ed8d6e725b55ef98b1b35c88013245f35f68b1b12c5cd4100dddac333",
-                "sha256:bb96fa6b56bb536c42d6a4a87dfca570ff8e52de2d63cabebfd6fb67049c34b6",
-                "sha256:bbcf1a76cf6f6dacf2c7f4d2ebd411438c275faa1dc0c68e46eb84eebd05dd7d",
-                "sha256:bca5f24726e2919de94f047739d0a4fc01372801a3672708260546aa2601bf57",
-                "sha256:bf2e1a9162c1e441bf805a1fd166e249d574ca04e03b34f97e2928769e91ab5c",
-                "sha256:c4eb3b82ca349cf6fadcdc7abcc8b3a50ab74a62e9113ab7a8ebc268aad35bb9",
-                "sha256:c6cc15d58053c76eacac5fa9152d7d84b8d67b3fde92709195cb984cfb3475ea",
-                "sha256:c6cd05ea06daca6ad6a4ca3ba7fe7dc5b5de063ff4daec6170ec0f9979f6c332",
-                "sha256:c844fd628851c0bc309f3c801b3a3d58ce430b2ce5b359cd918a5a76d0b20cb5",
-                "sha256:c9cb1565a7ad52e096a6988e2ee0397f72fe056dadf75d17fa6b5aebaea05622",
-                "sha256:cab9401de3ea52b4b4c6971db5fb5c999bd4260898af972bf23de1c6b5dd9d71",
-                "sha256:cd468460eefef601ece4428d3cf4562459157c0f6523db89365202c31b6daebb",
-                "sha256:d1e6a862b76f34395a985b3cd39a0d949ca80a70b6ebdea37d3ab39ceea6698a",
-                "sha256:d1f9282c5f2b5e241034a009779e7b2a1aa045f667ff521e7948ea9b56e0c5ff",
-                "sha256:d265f09a75a79a788237d7f9054f929ced2e69eb0bb79de3798c468d8a90f945",
-                "sha256:db3fc6120bce9f446d13b1b834ea5b15341ca9ff3f335e4a951a6ead31105480",
-                "sha256:dbf3a08a06b3f433013c143ebd72c15cac33d2914b8ea4bea7ac2c23578815d6",
-                "sha256:de04b491d0e5007ee1b63a309956eaed959a49f5bb4e84b26c8f5d49de140fa9",
-                "sha256:e4b09863aae0dc965c3ef36500d891a3ff495a2ea9ae9171e4519963c12ceefd",
-                "sha256:e595432ac259af2d4630008bf638873d69346372d38255774c0e286951e8b79f",
-                "sha256:e75b89ac3bd27d2d043b234aa7b734c38ba1b0e43f07787130a0ecac1e12228a",
-                "sha256:ea9eb976ffdd79d0e893869cfe179a8f60f152d42cb64622fca418cd9b18dc2a",
-                "sha256:eafb3e874816ebe2a92f5e155f17260034c8c341dad1df25672fb710627c6949",
-                "sha256:ee3c36df21b5714d49fc4580247947aa64bcbe2939d1b77b4c8dcb8f6c9faecc",
-                "sha256:f352b62b45dff37b55ddd7b9c0c8672c4dd2eb9c0f9c11d395075a84e2c40f75",
-                "sha256:fabb87dd8850ef0f7fe2b366d44b77d7e6fa2ea87861ab3844da99291e81e60f",
-                "sha256:fe11310ae1e4cd560035598c3f29d86cef39a83d244c7466f95c27ae04850f10",
-                "sha256:fe7ba4a51f33ab275515f66b0a236bcde4fb5561498fe8f898d4e549b2e4509f"
+                "sha256:00ad4b6f185ec67f3e6562e8a1d2b69660be43070bd0ef6fcec5211154c7df67",
+                "sha256:0175d745d9e85c40dcc51c8f88c74bfbaef9e7afeeeb9d03c37977270303064c",
+                "sha256:01d4c0c874aa4ddfb8098e85d10b5e875a70adc63db91f1ae65a4b04d3344cda",
+                "sha256:043d2299f6dfdc92f0ac5e995dfc56668e1587cea7f9aa9d8a78a1b6554e5755",
+                "sha256:0c413c633d0512df4dc7fd2373ec06cc6a815b7b6d6c2f208ada7e9e93a5061d",
+                "sha256:0d21c684808288a98914e5aaf2a7c6a3179d4df11d249799c32d1808e79503b5",
+                "sha256:0e584a10f204a617d71d359fe383406305a4b595b333721fa50b867b4a0a1548",
+                "sha256:1274477e4c71ce8cfe6c1ec2f806d57c015ebf84d83373676036e256bc55d690",
+                "sha256:13bf85afc99ce6f9ee3567b04501f18f9f8dbbb2ea11ed1a2e079670403a7c84",
+                "sha256:153c2549f6c004d2754cc60603d4668899c9895b8a89397444a9c4efa282aaf4",
+                "sha256:1f7372f7341fcc16f57b2caded43e81ddd18df53320b6f9f042acad41f8e049a",
+                "sha256:23fb25a9f0a1ca1f24c0a371523546366bb642397c94ab45ad3aedf2941cec6a",
+                "sha256:28c543e54710d6158fc6f439296c7865b29e0b616629767e685a7185fab4a6b9",
+                "sha256:2a482e6da906d5e6e653be079b29bc173a48e381600161c9932d89dfae5942ef",
+                "sha256:2ad5c3c4590bb3cc28b4382f031f3783f25ec223557124c68754a2231d989e2b",
+                "sha256:2ce2ac5708501afc4847221a521f7e4b245abf5178cf5ddae9d5b3856ddb2f3a",
+                "sha256:2cf57fb50be5f52bda004b8893e63b48530ed9f0d6c96c84620dc92fe3cd9b9d",
+                "sha256:2e1b1e51b0774408f091d268648e3d57f7260c1682e7d3a63cb00d22d71bb945",
+                "sha256:2e2e9839e14dd5308ee773c97115f1e0a1cb1d75cbeeee9f33824fa5144c7634",
+                "sha256:2e460be6978fc24e3df83193dc0cc4de46c9909ed92dd47d349a452ef49325b7",
+                "sha256:312fcfbacc7880a8da0ae8b6abc6cc7d752e9caa0051a53d217a650b25e9a691",
+                "sha256:33279701c04351a2914e1100b62b2a7fdb9a25995c4a104259f9a5ead7ed4802",
+                "sha256:33776e945d89b29251b33a7e7d006ce86447b2cfd66db5e5ded4e5cd0340585c",
+                "sha256:34dd0c107799dcbbf7d48b53be761a013c0adf5571bf50c4ecad5643fe9cfcd0",
+                "sha256:3562b06567c06439d8b447037bb655ef69786c590b1de86c7ab81efe1c9c15d8",
+                "sha256:368a42363c4d70ab52c2c6420a57f190ed3dfaca6a1b19afda8165ee16416a82",
+                "sha256:4149d34c32f9638f38f544b3977a4c24052042affa895352d3636fa8bffd030a",
+                "sha256:461908b2578955045efde733719d62f2b649c404189a09a632d245b445c9c975",
+                "sha256:4a01951fabc4ce26ab791da5f3f24dca6d9a6f24121746eb19756416ff2d881b",
+                "sha256:4e874cbf8caf8959d2adf572a78bba17cb0e9d7e51bb83d86a3697b686a0ab4d",
+                "sha256:4f21e83f355643c345177a5d1d8079f9f28b5133bcd154193b799d380331d5d3",
+                "sha256:5443910d662db951b2e58eb70b0fbe6b6e2ae613477129a5805d0b66c54b6cb7",
+                "sha256:5798a9aad1879f626589f3df0f8b79b3608a92e9beab10e5fda02c8a2c60db2e",
+                "sha256:5d20003b635fc6ae3f96d7260281dfaf1894fc3aa24d1888a9b2628e97c241e5",
+                "sha256:5db3a5b833764280ed7618393832e0853e40f3d3e9aa128ac0ba0f8278d08649",
+                "sha256:5ed1c46fb119f1b59304b5ec89f834f07124cd23ae5b74288e364477641060ff",
+                "sha256:62360cb771707cb70a6fd114b9871d20d7dd2163a0feafe43fd115cfe4fe845e",
+                "sha256:6809a00deaf3810e38c628e9a33271892f815b853605a936e2e9e5129762356c",
+                "sha256:68c5a82c8779bdfc6367c967a4a1b2aa52cd3595388bf5961a62158ee8a59e22",
+                "sha256:6e4a280e4b975a2e7745573e3fc9c9ba0d1194a3738ce1cbaa80626cc9b4f4df",
+                "sha256:6e6783bcc45f397fdebc118d772103d751b54cddf5b60fbcc958382d7dd64f3e",
+                "sha256:72a860c215e26192379f57cae5ab12b168b75db8271f111019509a1196dfc780",
+                "sha256:7607ec3ce4993464368505888af5beb446845a014bc676d349efec0e05085905",
+                "sha256:773dd01706d4db536335fcfae6ea2440a70ceb03dd3e7378f3e815b03c97ab51",
+                "sha256:78d847e4cde6ecc19125ccbc9bfac4a7ab37c234dd88fbb3c5c524e8e14da543",
+                "sha256:7dde0009408969a43b04c16cbbe252c4f5ef4574ac226bc8815cd7342d2028b6",
+                "sha256:80bd372b8d0715c66c974cf57fe363621a02f359f1ec81cba97366948c7fc873",
+                "sha256:841cd8233cbd2111a0ef0a522ce016357c5e3aff8a8ce92bcfa14cef890d698f",
+                "sha256:84de26ddf621d7ac4c975dbea4c945860e08cccde492269db4e1538a6a6f3c35",
+                "sha256:84f8ae3e09a34f35c18fa57f015cc394bd1389bce02503fb30c394d04ee6b938",
+                "sha256:8af740fc2711ad85f1a5c034a435782fbd5b5f8314c9a3ef071424a8158d7f6b",
+                "sha256:8b929b9bd7cd7c3939f8bcfffa92fae7480bd1aa425279d51a89327d600c704d",
+                "sha256:910bec0c49637d213f5d9877105d26e0c4a4de2f8b1b29405ff37e9fc0ad52b8",
+                "sha256:96943e5dcc37a6529d18766597c491798b7eb7a61d48878611298afc1fca946c",
+                "sha256:a0215ce6041d501f3155dc219712bc41252d0ab76474615b9700d63d4d9292af",
+                "sha256:a3cf433f127efa43fee6b90ea4c6edf6c4a17109d1d037d1a52abec84d8f2e42",
+                "sha256:a6ce61195c6a19c785df04e71a4537e29eaa2c50fe745b732aa937c0c77169f3",
+                "sha256:a7a75ef35f2df54ad55dbf4b73fe1da96f370e51b10c91f08b19603c64004acc",
+                "sha256:a94159871304770da4dd371f4291b20cac04e8c94f11bdea1c3478e557fbe0d8",
+                "sha256:aa1990247f02a54185dc0dff92a6904521172a22664c863a03ff64c42f9b5410",
+                "sha256:ab88bafedc57dd0aab55fa728ea10c1911f7e4d8b43e1d838a1739f33712921c",
+                "sha256:ad093e823df03bb3fd37e7dec9d4670c34f9e24aeace76808fc20a507cace825",
+                "sha256:ae871a964e1987a943d83d6709d20ec6103ca1eaf52f7e0d36ee1b5bebb8b9b9",
+                "sha256:b0ba0d15164eae3d878260d4c4df859bbdc6466e9e6689c344a13334f988bb53",
+                "sha256:b5411d82cddd212644cf9360879eb5080f0d5f7d809d03262c50dad02f01421a",
+                "sha256:b9552ec52cc147dbf1944ac7ac98af7602e51ea2dcd076ed194ca3c0d1c7d0bc",
+                "sha256:bfb9162dcf01f615462b995a516ba03e769de0789de1cadc0f916265c257e5d8",
+                "sha256:c0a9034379a37ae42dea7ac1e048352d96286626251862e448933c0f59cbd79c",
+                "sha256:c1161b345c0a444ebcf46bf0a740ba5dcf50612fd3d0528883fdc0eff578006a",
+                "sha256:c11f5b099adafb18e65c2c997d57108b5bbeaa9eeee64a84302c0978b1ec948b",
+                "sha256:c44e65da1de4403d0576473e2344828ef9c4c6244d65cf4b75549bb46d40b8dd",
+                "sha256:c48c5c0271149cfe467c0ff8eb941279fd6e3f65c9a388c984e0e6cf57538e14",
+                "sha256:c7a815258e5895d8900aec4454f38dca9aed71085f227537208057853f9d13f2",
+                "sha256:cae533195e8122584ec87531d6df000ad07737eaa3c81209e85c928854d2195c",
+                "sha256:cc14be025665dba6202b6a71cfcdb53210cc498e50068bc088076624471f8bb9",
+                "sha256:cd56db019015b6acfaaf92e1ac40eb8434847d9bf88b4be4efe5bfd260aee692",
+                "sha256:d827176898a2b0b09694fbd1088c7a31836d1a505c243811c87ae53a3f6273c1",
+                "sha256:df72ac063b97837a80d80dec8d54c241af059cc9bb42c4de68bd5b61ceb37caa",
+                "sha256:e5980a746d547a6ba173fd5ee85ce9077e72d118758db05d229044b469d9029a",
+                "sha256:e5d47ae48db0b2dcf70bc8a3bc72b3de86e2a590fc299fdbbb15af320d2659de",
+                "sha256:e91d635961bec2d8f19dfeb41a539eb94bd073f075ca6dae6c8dc0ee89ad6f91",
+                "sha256:ea353162f249c8097ea63c2169dd1aa55de1e8fecbe63412a9bc50816e87b761",
+                "sha256:eaeed7abfb5d64c539e2db173f63631455f1196c37d9d8d873fc316470dfbacd",
+                "sha256:eca4bf3734c541dc4f374ad6010a68ff6c6748f00451707f39857f429ca36ced",
+                "sha256:f83a552443a526ea38d064588613aca983d0ee0038801bc93c0c916428310c28",
+                "sha256:fb1558def481d84f03b45888473fc5a1f35747b5f334ef4e7a571bc0dfcb11f8",
+                "sha256:fd1ed388ea7fbed22c4968dd64bab0198de60750a25fe8c0c9d4bef5abe13824"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==3.8.4"
+            "version": "==3.8.5"
         },
         "aiosignal": {
             "hashes": [
@@ -162,14 +162,6 @@
             "markers": "python_version >= '3.7'",
             "version": "==23.1.0"
         },
-        "backoff": {
-            "hashes": [
-                "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba",
-                "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"
-            ],
-            "markers": "python_version >= '3.7' and python_version < '4.0'",
-            "version": "==2.2.1"
-        },
         "beautifulsoup4": {
             "hashes": [
                 "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30",
@@ -236,130 +228,130 @@
         },
         "bravado-core": {
             "hashes": [
-                "sha256:0da9c6f3814734622a55db3f62d08db6e188b25f3ebd087de370c91afb66a7f4",
-                "sha256:e231567cdc471337d23dfc950c45c5914ade8a78cde7ccf2ebb9433fcda29f40"
+                "sha256:1f40afc08ddc9199a2fe9732ea0695fac5fd8b871c10222e9718e8c13f91c88a",
+                "sha256:e3ac495fca5d475addeae45a2be8a51fd4120416d2183d630f001dfcfcc628dc"
             ],
-            "markers": "python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_full_version != '3.5.0'",
-            "version": "==5.17.1"
+            "markers": "python_version >= '3.7'",
+            "version": "==6.1.0"
         },
         "certifi": {
             "hashes": [
-                "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7",
-                "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"
+                "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082",
+                "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2023.5.7"
+            "version": "==2023.7.22"
         },
         "charset-normalizer": {
             "hashes": [
-                "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6",
-                "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1",
-                "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e",
-                "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373",
-                "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62",
-                "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230",
-                "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be",
-                "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c",
-                "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0",
-                "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448",
-                "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f",
-                "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649",
-                "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d",
-                "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0",
-                "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706",
-                "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a",
-                "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59",
-                "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23",
-                "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5",
-                "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb",
-                "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e",
-                "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e",
-                "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c",
-                "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28",
-                "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d",
-                "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41",
-                "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974",
-                "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce",
-                "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f",
-                "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1",
-                "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d",
-                "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8",
-                "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017",
-                "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31",
-                "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7",
-                "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8",
-                "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e",
-                "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14",
-                "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd",
-                "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d",
-                "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795",
-                "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b",
-                "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b",
-                "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b",
-                "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203",
-                "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f",
-                "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19",
-                "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1",
-                "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a",
-                "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac",
-                "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9",
-                "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0",
-                "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137",
-                "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f",
-                "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6",
-                "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5",
-                "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909",
-                "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f",
-                "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0",
-                "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324",
-                "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755",
-                "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb",
-                "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854",
-                "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c",
-                "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60",
-                "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84",
-                "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0",
-                "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b",
-                "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1",
-                "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531",
-                "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1",
-                "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11",
-                "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326",
-                "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df",
-                "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"
+                "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96",
+                "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c",
+                "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710",
+                "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706",
+                "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020",
+                "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252",
+                "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad",
+                "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329",
+                "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a",
+                "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f",
+                "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6",
+                "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4",
+                "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a",
+                "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46",
+                "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2",
+                "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23",
+                "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace",
+                "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd",
+                "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982",
+                "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10",
+                "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2",
+                "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea",
+                "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09",
+                "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5",
+                "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149",
+                "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489",
+                "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9",
+                "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80",
+                "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592",
+                "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3",
+                "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6",
+                "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed",
+                "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c",
+                "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200",
+                "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a",
+                "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e",
+                "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d",
+                "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6",
+                "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623",
+                "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669",
+                "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3",
+                "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa",
+                "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9",
+                "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2",
+                "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f",
+                "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1",
+                "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4",
+                "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a",
+                "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8",
+                "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3",
+                "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029",
+                "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f",
+                "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959",
+                "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22",
+                "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7",
+                "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952",
+                "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346",
+                "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e",
+                "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d",
+                "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299",
+                "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd",
+                "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a",
+                "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3",
+                "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037",
+                "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94",
+                "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c",
+                "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858",
+                "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a",
+                "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449",
+                "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c",
+                "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918",
+                "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1",
+                "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c",
+                "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac",
+                "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"
             ],
             "markers": "python_full_version >= '3.7.0'",
-            "version": "==3.1.0"
+            "version": "==3.2.0"
         },
         "click": {
             "hashes": [
-                "sha256:2739815aaa5d2c986a88f1e9230c55e17f0caad3d958a5e13ad0797c166db9e3",
-                "sha256:b97d0c74955da062a7d4ef92fadb583806a585b2ea81958a81bd72726cbb8e37"
+                "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd",
+                "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.1.4"
+            "version": "==8.1.6"
         },
         "cmake": {
             "hashes": [
-                "sha256:05cfd76c637eb22058c95e2dc383cadd4e0615e2643e637bb498a6cc24825790",
-                "sha256:1b92f9f59f48c803106dbdd6750b0f571a0500e25d3a62c42ba84bb7a9240d10",
-                "sha256:1d887be5f1a3f17559a78707a6bc0560f4f8cb93cebb9d823d90a63e68bae09b",
-                "sha256:230227bf99f36614de84cdc92ffce3a50eb2803020e946f8da945a08fcf766bf",
-                "sha256:235d8eac93a28dcce5a1cd7130412885a2aa53d5735cb2230e0f26f589347b65",
-                "sha256:24110035aff586a04a6a6fcf4609270642e4f503c0620c962dff75b653f81414",
-                "sha256:248a90816abfc10ff6e1109b54b8235c3e62f0ac92da16541753deb3b5ae063d",
-                "sha256:3175442985558d5415b97f264a6a1bb0af5ecfe10e3f7510257b1ea66bd33848",
-                "sha256:3e280e81713408987b7053f5b922c9f94e45668ca6efff1f02846309ca0b5b0f",
-                "sha256:432837364aa6cab2826a72e8a4cdd3586f5ac9ce495217ccd59aa70f2bba8120",
-                "sha256:93015da6f1c0e1e5f2debf752f1803ea52d742d915ad674043d36e471f937507",
-                "sha256:93a03bad17b9741acaff4a8651f8596496506602fa123e70fe67142f1b21ee2e",
-                "sha256:b7a6946c345497c14064e0c9585b30f5aaebbefdfc0b245b6bb5a978eb4fc85f",
-                "sha256:c3b0e72750c0f6c0373242c1299bc4ffdbebdd5004966ae6df0b2e9845aa6990",
-                "sha256:d45b30b9ce7280829888c78650177ab525df2b6785e1a5b3d82b4c147d828c0e",
-                "sha256:d726671ae7ae4aa6989e73d26b9f8f8e6af45163a26ea243949d72246566fdd8",
-                "sha256:e058e59154a1e490fb9425b420f87e28144292397607638d73e323509f7efae6"
-            ],
-            "version": "==3.26.4"
+                "sha256:073e4f196d0888216e6794c08cd984ddabc108c0e4e66f48fbd7610d1e6d726d",
+                "sha256:199bfaefb752e82d8067aeee5d6a6e0414fe0d60e9a3fd08e95d537a97e0db16",
+                "sha256:1b3189171665f5c8d748ae7fe10a29fff1ebeedeaef57b16f1ea54b1ec0fe514",
+                "sha256:1f38d87b2c65763a0113f4a6c652e6f4b5adf90b384c1e1d69e4f8a3104a57d6",
+                "sha256:35a8d397ce883e93b5e6561e2803ce9470df52283862264093c1078530f98189",
+                "sha256:48be3afe62c9513a49be007896a4058fafec512cb1f269a50126da30aacad97f",
+                "sha256:5561aca62b65aac844f3931e74cfeb696e4534de145e3307bf942e735736541e",
+                "sha256:58a3f39d3d1bc897f05e531bfa676246a2b25d424c6a47e4b6bbc193fb560db7",
+                "sha256:6f46a170b0c9c552d52da4346534570f818195dfc4f1d0c03264e24cc348fc60",
+                "sha256:8745eff805f36762d3e8e904698b853cb4a9da8b4b07d1c12bcd1e1a6c4a1709",
+                "sha256:9740ed9f61a3bd8708a41cadd5c057c04f38e5b89bd773e369df2e210a1c55a3",
+                "sha256:9ccab4cd93578d3c2df32e66b44b313b75a7484032645040431dc06a583ca4aa",
+                "sha256:b470ccd3f86cf19a63f6b221c9cceebcc58e32d3787d0d5f9f43d1d91a095090",
+                "sha256:b9d5811954dcedcaa6c915c4a9bb6d64b55ac189e9cbc74be726307d9d084804",
+                "sha256:c4c968c188e7518deb463a14e64f3a19f242c9dcf7f24e1dbcc1419690cd54e0",
+                "sha256:d03f0a76a2b96805044ad1178b92aeeb5f695caa6776a32522bb5c430a55b4e8",
+                "sha256:e58e48643903e6fad76274337f9a4d3c575b8e21cd05c6214780b2c98bb0c706"
+            ],
+            "version": "==3.27.0"
         },
         "colorama": {
             "hashes": [
@@ -379,68 +371,68 @@
         },
         "cramjam": {
             "hashes": [
-                "sha256:02a07a3e17fab7f1b1cf81c8fd80416bd06ca0a508cd8e379e280dc591641e14",
-                "sha256:02c0d09552137f07c8ea3681c215ce94e8431b7eaa5f293b747a24e9038c5d5c",
-                "sha256:076ce14ec4bc99b7de72e2da328b350d8e22d50a9a6880e0538863ef65d6d507",
-                "sha256:0a6a1df30ac4da907499bf9f160ee25947609e94f4c2093c6b1cb63698c61d17",
-                "sha256:0cfa188b95fb311892504df236f45029c03d8ac68634a6b5bb66487ee2d43f0e",
-                "sha256:1797a1367e9f854be2173e07812a096aec67e0e4891ce3c527d1536fb3023344",
-                "sha256:1ffdc8d1381b5fee57b33b537e38fa7fd29e8d8f3b544dbab1d71dbfaaec3bef",
-                "sha256:201924909735626eee4dcf841e720634ce753c3af30687c20958a0836221f6c2",
-                "sha256:2313e5353176b8659f9e157cc8ef64d7f6d80d381ae493abba0b6b213a8cb2ea",
-                "sha256:29769fbbb56cbada20ad66d0aa268a8f55dcef99053c5c16780394d5d656815a",
-                "sha256:2bd152234396942ca12d755f4dd5ab2e39f478c5900c290e4f9582bcc2290988",
-                "sha256:33d4bd5e280f055da306b0c78c72af4e166018bea3b38c50a44b6264188cfe10",
-                "sha256:346f4b3c7845ea1c5f0bc4e1c6cfd39153c399e3c03262d4c9e6575edb16c15a",
-                "sha256:35da290896b2953e3056441b6b42d3576588dddee28ae6a890b03047929ae34d",
-                "sha256:3a798f52a0cf6780f5b5aacc4a2ea06737f12b98762083d88c3e1ac6315260c7",
-                "sha256:4750df49a08396fbc06cf1fcf4055daa5e009f5e06e7fb5d70b23266f5bb28cc",
-                "sha256:489a16df91d10825ed47f95b985f8e353c8c8b4e078f571fd84e38a8ca95284b",
-                "sha256:4b322f268461e66530bfd851ae7e33eb37829284f6326d831b96eed1fbfee554",
-                "sha256:514afdeca432f1b870f4c15c51b2538f841ea36500d8b2b63c437e949a48d737",
-                "sha256:54c945fe1ab67bcd9ca90626176ec4fb74354c698e83c992641a5c4834dda675",
-                "sha256:60db085bff42099c4de9a2a0b10284ab4b1032d356193ada6275d3225dc16b0e",
-                "sha256:62c1ecc70be62e9dd5176949f2da6488f1e8981d33fd241a874f2f25f6fed3bf",
-                "sha256:63ba3affd35c780cd69382b022fade08b1b14e82f45aa86576e10b5520f21ffe",
-                "sha256:6455273781378befa00d096d9a58bcaee2c34f59057149220dd8edd185627b59",
-                "sha256:656505c16ece09d54a98b0128d0ce3e75a09ed27aafa9fc36c6881b736f9740b",
-                "sha256:664a1ae58d735c92551cfbbc09d9398bb218c7e6833b2392fece71af1dcbeedd",
-                "sha256:6a37eab7095c4bd4ae81d5c8c16c8fcf0e5c1c85c732716e2e7d6fdd873fc756",
-                "sha256:6b78702dbc1a4b1f4da613c63c7be578d418a561025432e1e0400b0274800917",
-                "sha256:6b98fb0ebc565298fab857bc09273c0fb57167a2703c51436f7f423ca62b8009",
-                "sha256:70c725e90d7beede63e663a335eb8adf2038a5b1a5f5ae32fcfa25cda164b520",
-                "sha256:755cdc43d79de9da949be797ee6f773a85cdec493f0339f48d944ebb7cc9342e",
-                "sha256:76db0782d610644be01a6aabad16d51a5989c58a07b27353b7c10ce1fe8cdfd3",
-                "sha256:797beb5629e814766b6ebbc73625a6f741739ca302ec1bcb12b47e39e8a1e4d7",
-                "sha256:7a6b474523eb23d090812ace971ce28297664913b0d9b753f6648c766af7dc7e",
-                "sha256:83041d02a9c3f09a41d5687f0a5dd2e5e591c6f5d7ccceba2d4788adf58dccb7",
-                "sha256:88f847313b6b3c4191eb15f74f3354f126297b7a51773cbfcc6a2917ecdcf40e",
-                "sha256:9034e7591fd689f17b5de830026a75f9c1788f0c78416a0845ba4c91cf4e896c",
-                "sha256:9928cb6703209c9a5474863973d09ab3c5cfbc10c051acec9af917413f64026b",
-                "sha256:a15db0c29278517465eee33bb5a4637930673ead242c98c81e613de0486ed00d",
-                "sha256:a4b9aa1eefb690a16bab8aaa522d6e4595b86a0e4924f062ec261771061434c5",
-                "sha256:a6df618d5f4b4d09ccde28beb5b1e6806ff88efbfa7d67e47226afa84363db51",
-                "sha256:a89a48527cf416a7b4fcd97e924fa8784b51ec4c38911c4454663648b4a4914f",
-                "sha256:a9add823575629c026625080b92ff44ba7bb6ade7f661252a07e6b300e1a689b",
-                "sha256:b6adc0f2f2d1c4fd0f93ecef036a3217b785737ada3bc7ad4233db6ca98eafff",
-                "sha256:b772394920b48af69db4b7b3b2e2684524f4b6d73a8e8e595811e2cc9d2fbee5",
-                "sha256:c3a55e68ed847b5fd8d151f9998d150d47d689cedbf89c11c0c05db656fd6336",
-                "sha256:d1d3d73125fa692974ccf0c0af4a444564e52c45b933527d4c047630b8c4b78f",
-                "sha256:d2256470f96c3ab0aba3b45f62708ea8eb98603f3417d9d1d3bd5cb4140fbf56",
-                "sha256:d2b621ce7d92a6620fb1737892b7b00a5911d92f80f0d5b454795ba1cd844e51",
-                "sha256:ddc0776fc98cbd7967a3c243666363eb88e5d32c2e9640b8f59f4f5cd2934161",
-                "sha256:ea42c629046964bcfa9971d0a978fb647d769f726f69ad39a8c6c5dc435616ad",
-                "sha256:edc655ec9014d5ebe827d578e03c0ae2839b05fba6dcddf412059e3f7e4a3a68",
-                "sha256:f09dbb5cde9d4130677875c68e53568829e00bda3eb85b880190e8c56ba7af73",
-                "sha256:f0c83c643b1fe6a79a938e453c139c73a196182d47915897b2cbadf46531a3a5",
-                "sha256:f45b2a6776637edd5017c5c1c6a167243a8d2250698e10971ce8da015ed43442",
-                "sha256:f8bc899d401055d7e54ce0182eda303c80f22c4a3271a01f44c72d51a07c4fed",
-                "sha256:fa5ae1851a9fa93c3d6f1f2051d2a51438479476f2a07dd0f04e47d23ceea708",
-                "sha256:fa655603b0cf88e029f4d328d10bb6b78866a388c8bda4e5d17b5b644827d8cf",
-                "sha256:fe176dcb03ec241c48b6af4be800f3e99f6b5be52ea2b660511374be709be926"
+                "sha256:01573d3c05f75845af75f87d9f6d4d995ee48fafc9c9662679d944c71e658c59",
+                "sha256:03a72a689c93b8a2c7c08b529c1224c47bd469722e559af231016694b90f6442",
+                "sha256:0b48b11c328d91250dadc63c00753f5ba26eb9df5fe75ba2ce8a78631260479d",
+                "sha256:0ffb891294e77f2a3b0137992ebd6eb9b1f1bc3728d7d4314632e30270855117",
+                "sha256:19c44de9dee2ea0c586a5b12adc9bc28224544717bce88a94c3ee202b9ece25d",
+                "sha256:1bfd9e9f050a4ef82644a5443742bd6e4d41afc201d5142dfbb5ac73a4f96b94",
+                "sha256:202e4a5a496ea49d0bb5252fbee8b6e421d256968773c7a8b3e86d98eec9228e",
+                "sha256:22927dbdda85d8719074e061f9ec024c9bf16088e5e4c6c1c134c46e2d9153b7",
+                "sha256:241b9853de56524895c17541d34bdcdd316261247253cb9faeb57a21cc2ae28c",
+                "sha256:2645f0e01735231e3c4d1568c95cc00ed477c1c2e2ed45cb123bef2ccbe72282",
+                "sha256:272ab041e95e4088321bffd63ead85d32f86f9fc79f37944663abb59e7fecbb1",
+                "sha256:2a96ff622f0db7f89d7c9aeadd9cc0c9bf61e804841a03a22ca919aa4955640d",
+                "sha256:3d5ed0fa20b42e063ef66ad01d9948e868bbfc327bf86604e078b67f074f76f3",
+                "sha256:3f2e41dc8143d0e88ec9ba3ff66c8d2aea486b04931119abe9e394f9961d74bc",
+                "sha256:44e262f083a0d1c19bc7b8bf8aaa54d31653eab67762cdaeb68fb40b844d50b9",
+                "sha256:44fe99233ef2f42ff03d8395e7d97e0c45306eb356f6f01fa69bdb49783fdb8a",
+                "sha256:47ae97247a58d3095be800420bba7e43cc6958e67f9dfddd12decdb4c99c4d6f",
+                "sha256:4d56afb5f278a18743a218514825b6ab176f18a4084d8f6515c64e3acef19478",
+                "sha256:4f48e35701528beed092d97c5a1a6ba6c8902c3485d7e3a55c463162bb66afa0",
+                "sha256:53ac8df43546d3f70acbc5c17424a8c083418bd6d2cacfbd5108aaa8f3eb26db",
+                "sha256:579fb724eec048b1a18ca8f7ad9a7ef296dc02eba5f87fd4d5031f0c32c5c9ac",
+                "sha256:58bf5b4dd8f1edaa3bd66b51a26790115efa36cebf66a198a60d095739010b36",
+                "sha256:5e3869a2ecb7764f18e81afa07aa15a6f28921ae4508ab002314aafecfe7a285",
+                "sha256:5f8d684a8b918e289bd213d55ea5f67dbf6cc379492e2a7e094724ec62846dea",
+                "sha256:60951e64d3e05ef2a46d2a92fc4e4563ae5e28bb3b6f231f2dca68a5078a72dc",
+                "sha256:66d6a7906fbe8c2e46e987d5a3a9cb235e931b3e4721ac7d8573151c419e3f73",
+                "sha256:6a8f0fd14bd11f2a625e5c554fd93c7775082c31ffd9fbabb4fe9db3031645d0",
+                "sha256:71bf6a6632648333c402a8692fe61f45416066eb0d8b7f4530cdf37fee221a11",
+                "sha256:79c36d95e89b43c29595c889c7a4d30d29aefc55d7c58a26a058b9bbe7abd5cf",
+                "sha256:7b0fa1f8600fdf093a4e5eac5dc56084047abec50d589a8121618e3fb1f9de3f",
+                "sha256:7e44dda432a8c8a47cb73869201d2f9777604bd913d859def84c659fb736cfd3",
+                "sha256:820421600d395d0e11b9da1e983acde453637090f5a15abb45f5182cd35f9c3a",
+                "sha256:89b683919ffc846b91b405d5f14df8a2ba1167f4ed9277150298fbe91a3d4aae",
+                "sha256:8ea1c781d3760df0b6ad80b7b19dc8e038e0638fb1cfabc68da96cedb8d0adca",
+                "sha256:91bb03ca0d3857f319afd13525d5c6214a55aa49778ce46a02c16f0eee37907c",
+                "sha256:9698ed7fe78b4de81dcee0624311ee93a4436f0b3e3c6b147a6745a3fd210272",
+                "sha256:a08dcb7c7b54f82db4ee9120aaace06326499c0d4108770ee7ac63d7bd1d803d",
+                "sha256:a7f93316abc1abfd348b04afc6cadbbd4fba44cd91e7b9803c9330045a7a1885",
+                "sha256:aac9d49e16f473ceb4eaf74a53180eac3363127f01855c39122b400a988e80bf",
+                "sha256:bee04fd1cdd5f2a2e91e4b271f22e228c698fe7b7f8ef209374d717f7889e80c",
+                "sha256:bef07e7d4607c4d70627e58eb630fe60e48b80a61ab05b33314e3296eb90af78",
+                "sha256:c2602c42ed101ada634fa37253d40946f0468b2b749689781cba715a7d78038e",
+                "sha256:ca6309d06edf29a9a5c22e25d1d7f7609abb8ae7281583bc486afb19fd645898",
+                "sha256:d07c5af763501bd23523658aeb535082eaac014746f7973df85f76b0d9b40967",
+                "sha256:d143ccb3f5aae640f0c2dd6244d43f5e5e81d847b50c2eb9f08dcc3dc33f367a",
+                "sha256:d5c411d785cec410d4164e4ecc76b6c152761fbb61325bcc4acbdc8926874c0b",
+                "sha256:dd77e784289fc44a5e6487b2e263f07cc271f25a8395e97213b6a934fe47a768",
+                "sha256:df46019cb999d874ce86e08d71d7d2983c052d6a63f7aa6bce960e4e05e8ea37",
+                "sha256:e6231fd3ac680c34c0d8405abfa8c3d12f92e28d0897d960aa905f053cc09e63",
+                "sha256:e8b8b968db6a6ec03de231ca17536f8cd69d74e36f0e47e68391a22231c0042c",
+                "sha256:ebc87176851c50c5aaf6bacafb6bed5a86e3b4ee6a749d6ec13f3d37ae0e951a",
+                "sha256:ee4bc46e5cf87d6097833bca33a66f2724b4773242a71ed642d13682fedefb71",
+                "sha256:ee80ebd85acec1031e7563cce3de3961bd3f2ec8947c5bf84a9356b25af67a75",
+                "sha256:f3544ea95d0e98ac926d92d652adc417e78091117cbe2ef7733e26c40601604c",
+                "sha256:f52718e1c7aed3d0e3ffbbe5c085d9c449daa726379788ddb27cb62ffc2b6ba1",
+                "sha256:f9340c70d95ea102cf51a07ecc09f93f1363e585d97b5276734a5f8c4476e560",
+                "sha256:fab800ed93fb129d85c63f26660b695fb194efb29765a163f269321778e28a8d",
+                "sha256:fbc59b223750a901d65d96333461ab17076594fa34448ed2ef911bd4b628f068",
+                "sha256:fbffb1f63edf4cb4272a25de288c2f2e20914bb93e003883656774e61794b960"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.6.2"
+            "version": "==2.7.0"
         },
         "datasets": {
             "hashes": [
@@ -452,10 +444,10 @@
         },
         "deepspeed": {
             "hashes": [
-                "sha256:a3743db45066afb67fde562e586e92ee5afd8f5ab4c390e6f796f5ec1b112290"
+                "sha256:afb06a97fde2a33d0cbd60a8357a70087c037b9f647ca48377728330c35eff3e"
             ],
             "index": "pypi",
-            "version": "==0.9.5"
+            "version": "==0.10.0"
         },
         "dill": {
             "hashes": [
@@ -465,6 +457,12 @@
             "index": "pypi",
             "version": "==0.3.4"
         },
+        "docopt": {
+            "hashes": [
+                "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"
+            ],
+            "version": "==0.6.2"
+        },
         "einops": {
             "hashes": [
                 "sha256:99149e46cc808956b174932fe563d920db4d6e5dadb8c6ecdaa7483b7ef7cfc3",
@@ -536,83 +534,70 @@
         },
         "frozenlist": {
             "hashes": [
-                "sha256:008a054b75d77c995ea26629ab3a0c0d7281341f2fa7e1e85fa6153ae29ae99c",
-                "sha256:02c9ac843e3390826a265e331105efeab489ffaf4dd86384595ee8ce6d35ae7f",
-                "sha256:034a5c08d36649591be1cbb10e09da9f531034acfe29275fc5454a3b101ce41a",
-                "sha256:05cdb16d09a0832eedf770cb7bd1fe57d8cf4eaf5aced29c4e41e3f20b30a784",
-                "sha256:0693c609e9742c66ba4870bcee1ad5ff35462d5ffec18710b4ac89337ff16e27",
-                "sha256:0771aed7f596c7d73444c847a1c16288937ef988dc04fb9f7be4b2aa91db609d",
-                "sha256:0af2e7c87d35b38732e810befb9d797a99279cbb85374d42ea61c1e9d23094b3",
-                "sha256:14143ae966a6229350021384870458e4777d1eae4c28d1a7aa47f24d030e6678",
-                "sha256:180c00c66bde6146a860cbb81b54ee0df350d2daf13ca85b275123bbf85de18a",
-                "sha256:1841e200fdafc3d51f974d9d377c079a0694a8f06de2e67b48150328d66d5483",
-                "sha256:23d16d9f477bb55b6154654e0e74557040575d9d19fe78a161bd33d7d76808e8",
-                "sha256:2b07ae0c1edaa0a36339ec6cce700f51b14a3fc6545fdd32930d2c83917332cf",
-                "sha256:2c926450857408e42f0bbc295e84395722ce74bae69a3b2aa2a65fe22cb14b99",
-                "sha256:2e24900aa13212e75e5b366cb9065e78bbf3893d4baab6052d1aca10d46d944c",
-                "sha256:303e04d422e9b911a09ad499b0368dc551e8c3cd15293c99160c7f1f07b59a48",
-                "sha256:352bd4c8c72d508778cf05ab491f6ef36149f4d0cb3c56b1b4302852255d05d5",
-                "sha256:3843f84a6c465a36559161e6c59dce2f2ac10943040c2fd021cfb70d58c4ad56",
-                "sha256:394c9c242113bfb4b9aa36e2b80a05ffa163a30691c7b5a29eba82e937895d5e",
-                "sha256:3bbdf44855ed8f0fbcd102ef05ec3012d6a4fd7c7562403f76ce6a52aeffb2b1",
-                "sha256:40de71985e9042ca00b7953c4f41eabc3dc514a2d1ff534027f091bc74416401",
-                "sha256:41fe21dc74ad3a779c3d73a2786bdf622ea81234bdd4faf90b8b03cad0c2c0b4",
-                "sha256:47df36a9fe24054b950bbc2db630d508cca3aa27ed0566c0baf661225e52c18e",
-                "sha256:4ea42116ceb6bb16dbb7d526e242cb6747b08b7710d9782aa3d6732bd8d27649",
-                "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a",
-                "sha256:5c11e43016b9024240212d2a65043b70ed8dfd3b52678a1271972702d990ac6d",
-                "sha256:5cf820485f1b4c91e0417ea0afd41ce5cf5965011b3c22c400f6d144296ccbc0",
-                "sha256:5d8860749e813a6f65bad8285a0520607c9500caa23fea6ee407e63debcdbef6",
-                "sha256:6327eb8e419f7d9c38f333cde41b9ae348bec26d840927332f17e887a8dcb70d",
-                "sha256:65a5e4d3aa679610ac6e3569e865425b23b372277f89b5ef06cf2cdaf1ebf22b",
-                "sha256:66080ec69883597e4d026f2f71a231a1ee9887835902dbe6b6467d5a89216cf6",
-                "sha256:783263a4eaad7c49983fe4b2e7b53fa9770c136c270d2d4bbb6d2192bf4d9caf",
-                "sha256:7f44e24fa70f6fbc74aeec3e971f60a14dde85da364aa87f15d1be94ae75aeef",
-                "sha256:7fdfc24dcfce5b48109867c13b4cb15e4660e7bd7661741a391f821f23dfdca7",
-                "sha256:810860bb4bdce7557bc0febb84bbd88198b9dbc2022d8eebe5b3590b2ad6c842",
-                "sha256:841ea19b43d438a80b4de62ac6ab21cfe6827bb8a9dc62b896acc88eaf9cecba",
-                "sha256:84610c1502b2461255b4c9b7d5e9c48052601a8957cd0aea6ec7a7a1e1fb9420",
-                "sha256:899c5e1928eec13fd6f6d8dc51be23f0d09c5281e40d9cf4273d188d9feeaf9b",
-                "sha256:8bae29d60768bfa8fb92244b74502b18fae55a80eac13c88eb0b496d4268fd2d",
-                "sha256:8df3de3a9ab8325f94f646609a66cbeeede263910c5c0de0101079ad541af332",
-                "sha256:8fa3c6e3305aa1146b59a09b32b2e04074945ffcfb2f0931836d103a2c38f936",
-                "sha256:924620eef691990dfb56dc4709f280f40baee568c794b5c1885800c3ecc69816",
-                "sha256:9309869032abb23d196cb4e4db574232abe8b8be1339026f489eeb34a4acfd91",
-                "sha256:9545a33965d0d377b0bc823dcabf26980e77f1b6a7caa368a365a9497fb09420",
-                "sha256:9ac5995f2b408017b0be26d4a1d7c61bce106ff3d9e3324374d66b5964325448",
-                "sha256:9bbbcedd75acdfecf2159663b87f1bb5cfc80e7cd99f7ddd9d66eb98b14a8411",
-                "sha256:a4ae8135b11652b08a8baf07631d3ebfe65a4c87909dbef5fa0cdde440444ee4",
-                "sha256:a6394d7dadd3cfe3f4b3b186e54d5d8504d44f2d58dcc89d693698e8b7132b32",
-                "sha256:a97b4fe50b5890d36300820abd305694cb865ddb7885049587a5678215782a6b",
-                "sha256:ae4dc05c465a08a866b7a1baf360747078b362e6a6dbeb0c57f234db0ef88ae0",
-                "sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530",
-                "sha256:b1e2c1185858d7e10ff045c496bbf90ae752c28b365fef2c09cf0fa309291669",
-                "sha256:b4395e2f8d83fbe0c627b2b696acce67868793d7d9750e90e39592b3626691b7",
-                "sha256:b756072364347cb6aa5b60f9bc18e94b2f79632de3b0190253ad770c5df17db1",
-                "sha256:ba64dc2b3b7b158c6660d49cdb1d872d1d0bf4e42043ad8d5006099479a194e5",
-                "sha256:bed331fe18f58d844d39ceb398b77d6ac0b010d571cba8267c2e7165806b00ce",
-                "sha256:c188512b43542b1e91cadc3c6c915a82a5eb95929134faf7fd109f14f9892ce4",
-                "sha256:c21b9aa40e08e4f63a2f92ff3748e6b6c84d717d033c7b3438dd3123ee18f70e",
-                "sha256:ca713d4af15bae6e5d79b15c10c8522859a9a89d3b361a50b817c98c2fb402a2",
-                "sha256:cd4210baef299717db0a600d7a3cac81d46ef0e007f88c9335db79f8979c0d3d",
-                "sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9",
-                "sha256:d5cd3ab21acbdb414bb6c31958d7b06b85eeb40f66463c264a9b343a4e238642",
-                "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0",
-                "sha256:e235688f42b36be2b6b06fc37ac2126a73b75fb8d6bc66dd632aa35286238703",
-                "sha256:eb82dbba47a8318e75f679690190c10a5e1f447fbf9df41cbc4c3afd726d88cb",
-                "sha256:ebb86518203e12e96af765ee89034a1dbb0c3c65052d1b0c19bbbd6af8a145e1",
-                "sha256:ee78feb9d293c323b59a6f2dd441b63339a30edf35abcb51187d2fc26e696d13",
-                "sha256:eedab4c310c0299961ac285591acd53dc6723a1ebd90a57207c71f6e0c2153ab",
-                "sha256:efa568b885bca461f7c7b9e032655c0c143d305bf01c30caf6db2854a4532b38",
-                "sha256:efce6ae830831ab6a22b9b4091d411698145cb9b8fc869e1397ccf4b4b6455cb",
-                "sha256:f163d2fd041c630fed01bc48d28c3ed4a3b003c00acd396900e11ee5316b56bb",
-                "sha256:f20380df709d91525e4bee04746ba612a4df0972c1b8f8e1e8af997e678c7b81",
-                "sha256:f30f1928162e189091cf4d9da2eac617bfe78ef907a761614ff577ef4edfb3c8",
-                "sha256:f470c92737afa7d4c3aacc001e335062d582053d4dbe73cda126f2d7031068dd",
-                "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"
+                "sha256:007df07a6e3eb3e33e9a1fe6a9db7af152bbd8a185f9aaa6ece10a3529e3e1c6",
+                "sha256:008eb8b31b3ea6896da16c38c1b136cb9fec9e249e77f6211d479db79a4eaf01",
+                "sha256:09163bdf0b2907454042edb19f887c6d33806adc71fbd54afc14908bfdc22251",
+                "sha256:0c7c1b47859ee2cac3846fde1c1dc0f15da6cec5a0e5c72d101e0f83dcb67ff9",
+                "sha256:0e5c8764c7829343d919cc2dfc587a8db01c4f70a4ebbc49abde5d4b158b007b",
+                "sha256:10ff5faaa22786315ef57097a279b833ecab1a0bfb07d604c9cbb1c4cdc2ed87",
+                "sha256:17ae5cd0f333f94f2e03aaf140bb762c64783935cc764ff9c82dff626089bebf",
+                "sha256:19488c57c12d4e8095a922f328df3f179c820c212940a498623ed39160bc3c2f",
+                "sha256:1a0848b52815006ea6596c395f87449f693dc419061cc21e970f139d466dc0a0",
+                "sha256:1e78fb68cf9c1a6aa4a9a12e960a5c9dfbdb89b3695197aa7064705662515de2",
+                "sha256:261b9f5d17cac914531331ff1b1d452125bf5daa05faf73b71d935485b0c510b",
+                "sha256:2b8bcf994563466db019fab287ff390fffbfdb4f905fc77bc1c1d604b1c689cc",
+                "sha256:38461d02d66de17455072c9ba981d35f1d2a73024bee7790ac2f9e361ef1cd0c",
+                "sha256:490132667476f6781b4c9458298b0c1cddf237488abd228b0b3650e5ecba7467",
+                "sha256:491e014f5c43656da08958808588cc6c016847b4360e327a62cb308c791bd2d9",
+                "sha256:515e1abc578dd3b275d6a5114030b1330ba044ffba03f94091842852f806f1c1",
+                "sha256:556de4430ce324c836789fa4560ca62d1591d2538b8ceb0b4f68fb7b2384a27a",
+                "sha256:5833593c25ac59ede40ed4de6d67eb42928cca97f26feea219f21d0ed0959b79",
+                "sha256:6221d84d463fb110bdd7619b69cb43878a11d51cbb9394ae3105d082d5199167",
+                "sha256:6918d49b1f90821e93069682c06ffde41829c346c66b721e65a5c62b4bab0300",
+                "sha256:6c38721585f285203e4b4132a352eb3daa19121a035f3182e08e437cface44bf",
+                "sha256:71932b597f9895f011f47f17d6428252fc728ba2ae6024e13c3398a087c2cdea",
+                "sha256:7211ef110a9194b6042449431e08c4d80c0481e5891e58d429df5899690511c2",
+                "sha256:764226ceef3125e53ea2cb275000e309c0aa5464d43bd72abd661e27fffc26ab",
+                "sha256:7645a8e814a3ee34a89c4a372011dcd817964ce8cb273c8ed6119d706e9613e3",
+                "sha256:76d4711f6f6d08551a7e9ef28c722f4a50dd0fc204c56b4bcd95c6cc05ce6fbb",
+                "sha256:7f4f399d28478d1f604c2ff9119907af9726aed73680e5ed1ca634d377abb087",
+                "sha256:88f7bc0fcca81f985f78dd0fa68d2c75abf8272b1f5c323ea4a01a4d7a614efc",
+                "sha256:8d0edd6b1c7fb94922bf569c9b092ee187a83f03fb1a63076e7774b60f9481a8",
+                "sha256:901289d524fdd571be1c7be054f48b1f88ce8dddcbdf1ec698b27d4b8b9e5d62",
+                "sha256:93ea75c050c5bb3d98016b4ba2497851eadf0ac154d88a67d7a6816206f6fa7f",
+                "sha256:981b9ab5a0a3178ff413bca62526bb784249421c24ad7381e39d67981be2c326",
+                "sha256:9ac08e601308e41eb533f232dbf6b7e4cea762f9f84f6357136eed926c15d12c",
+                "sha256:a02eb8ab2b8f200179b5f62b59757685ae9987996ae549ccf30f983f40602431",
+                "sha256:a0c6da9aee33ff0b1a451e867da0c1f47408112b3391dd43133838339e410963",
+                "sha256:a6c8097e01886188e5be3e6b14e94ab365f384736aa1fca6a0b9e35bd4a30bc7",
+                "sha256:aa384489fefeb62321b238e64c07ef48398fe80f9e1e6afeff22e140e0850eef",
+                "sha256:ad2a9eb6d9839ae241701d0918f54c51365a51407fd80f6b8289e2dfca977cc3",
+                "sha256:b206646d176a007466358aa21d85cd8600a415c67c9bd15403336c331a10d956",
+                "sha256:b826d97e4276750beca7c8f0f1a4938892697a6bcd8ec8217b3312dad6982781",
+                "sha256:b89ac9768b82205936771f8d2eb3ce88503b1556324c9f903e7156669f521472",
+                "sha256:bd7bd3b3830247580de99c99ea2a01416dfc3c34471ca1298bccabf86d0ff4dc",
+                "sha256:bdf1847068c362f16b353163391210269e4f0569a3c166bc6a9f74ccbfc7e839",
+                "sha256:c11b0746f5d946fecf750428a95f3e9ebe792c1ee3b1e96eeba145dc631a9672",
+                "sha256:c5374b80521d3d3f2ec5572e05adc94601985cc526fb276d0c8574a6d749f1b3",
+                "sha256:ca265542ca427bf97aed183c1676e2a9c66942e822b14dc6e5f42e038f92a503",
+                "sha256:ce31ae3e19f3c902de379cf1323d90c649425b86de7bbdf82871b8a2a0615f3d",
+                "sha256:ceb6ec0a10c65540421e20ebd29083c50e6d1143278746a4ef6bcf6153171eb8",
+                "sha256:d081f13b095d74b67d550de04df1c756831f3b83dc9881c38985834387487f1b",
+                "sha256:d5655a942f5f5d2c9ed93d72148226d75369b4f6952680211972a33e59b1dfdc",
+                "sha256:d5a32087d720c608f42caed0ef36d2b3ea61a9d09ee59a5142d6070da9041b8f",
+                "sha256:d6484756b12f40003c6128bfcc3fa9f0d49a687e171186c2d85ec82e3758c559",
+                "sha256:dd65632acaf0d47608190a71bfe46b209719bf2beb59507db08ccdbe712f969b",
+                "sha256:de343e75f40e972bae1ef6090267f8260c1446a1695e77096db6cfa25e759a95",
+                "sha256:e29cda763f752553fa14c68fb2195150bfab22b352572cb36c43c47bedba70eb",
+                "sha256:e41f3de4df3e80de75845d3e743b3f1c4c8613c3997a912dbf0229fc61a8b963",
+                "sha256:e66d2a64d44d50d2543405fb183a21f76b3b5fd16f130f5c99187c3fb4e64919",
+                "sha256:e74b0506fa5aa5598ac6a975a12aa8928cbb58e1f5ac8360792ef15de1aa848f",
+                "sha256:f0ed05f5079c708fe74bf9027e95125334b6978bf07fd5ab923e9e55e5fbb9d3",
+                "sha256:f61e2dc5ad442c52b4887f1fdc112f97caeff4d9e6ebe78879364ac59f1663e1",
+                "sha256:fec520865f42e5c7f050c2a79038897b1c7d1595e907a9e08e3353293ffc948e"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.3.3"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.4.0"
         },
         "fsspec": {
             "extras": [
@@ -642,11 +627,11 @@
         },
         "gitpython": {
             "hashes": [
-                "sha256:8ce3bcf69adfdf7c7d503e78fd3b1c492af782d58893b650adb2ac8912ddd573",
-                "sha256:f04893614f6aa713a60cbbe1e6a97403ef633103cdd0ef5eb6efe0deb98dbe8d"
+                "sha256:8d9b8cb1e80b9735e8717c9362079d3ce4c6e5ddeebedd0361b228c3a67a62f6",
+                "sha256:e3d59b1c2c6ebb9dfa7a184daf3b6dd4914237e7488a1730a6d8f6f5d0b4187f"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.1.31"
+            "version": "==3.1.32"
         },
         "gputil": {
             "hashes": [
@@ -731,15 +716,40 @@
         },
         "h2o-wave": {
             "hashes": [
-                "sha256:4b749ae79558f9d9cf5b6fa4ea1e06a4ba4353976a5c48d1536d26feed9b4d12",
-                "sha256:591160922dc67023a9233611a5708d830dce919a4069f7a2c7f4e5b27d6d9bcb",
-                "sha256:5ea8843f97636e190ac36437c978db00ba38911838d839de52d2405877a7cb71",
-                "sha256:86e118991de4dc441ac22a863f97a7ec3a1f897ea41fc955e2da96ec5c1ad6d7",
-                "sha256:ef952af629f9fb8c9c2b4adceb0deb4bbd0d35de5097e1238bfc203129bfe870",
-                "sha256:f8ddd0a69da26963302502591166c2302d9eba4a0ed63916052c5b6c4849f6ac"
+                "sha256:211949ce2a9cb5ffb92fdff545dc538c55a2eba94bf864b5e88769bd066dbfa7",
+                "sha256:5bfec5caf1b1402420546a6a1ededfbdbcd5253a813da60f0bc71d9cd04b1358",
+                "sha256:8740e52916aaa4b8f4a62236db2c8e9898b427f07245261493510c9ab090f7f7",
+                "sha256:dbac4903ba8606acd1ccb2bb215e8582ac3a3b72edc4f8acf41e309735f19696",
+                "sha256:e0ef6fddf80f3447518e65daa54081ca83cfa48f7aead70a54320c0a5041a578",
+                "sha256:eb21c4c5c95a4f38d38592586ba6ae47bcbdcb08c4375d32e242cde422ab1f70"
             ],
             "index": "pypi",
-            "version": "==0.26.1"
+            "version": "==0.26.2"
+        },
+        "hf-transfer": {
+            "hashes": [
+                "sha256:0c1bdfa554f0b0936c1623b99058c6998a00fdcd86f75d9203f3f66572d2e30c",
+                "sha256:334862f4a82f8a09d6d3f550e67d7e498bb8882e678b7725638254fed3276801",
+                "sha256:3ce6c5965a57d94db5e043aa488a4df929a32000db125d9c9a1d325e8c7006dc",
+                "sha256:445edfcd9d59c9d2541957177a9c9225b1f0e8855f6311fb16e20f67c3426421",
+                "sha256:569ef1ec6fec182e706ade4ea0c63f8510fd618ed7ced7c772efaafac7245b07",
+                "sha256:606f2fe436e5be73f07987a56cd97c695805413d29203ae39ebd9fc596405435",
+                "sha256:617692a70cf299576d82cfc860923f29ec5c834a3f5242bc0853d4f106670398",
+                "sha256:6e5201b648df6106c232fcdb507db734081fd6220dfb1c432bd27c6fa9453331",
+                "sha256:79099ac043423b263a2843a24213418f309d5c8bc458776622bffe012ebced73",
+                "sha256:7afd7eb03efad7812a48591b639b2e3f3d1f93c1e9060c18cc63ebf08d7e193c",
+                "sha256:862b6ddba8e236bdc73408c20d020cfe5069cac3fd0b6de901c46f031df2b7d9",
+                "sha256:8a7934c8e491bb395731f677f66dd5f6641432f338a3a9efc9f0b6c186d37cf8",
+                "sha256:9913f5ad2644a1f57c1b7755a7d959ca5e0189863bb0473817d0707af230bf6a",
+                "sha256:c9faa88b3491c50d4aa75faf18ae24040cd91aa0565c7f7ba2357dbcbf8372f6",
+                "sha256:ca218fb6efc5c22379d9e64989e84bff426fcf5664fdbbf7cd70aa8b79497652",
+                "sha256:d53954ecfaadc84c15481bf5d4c7282323196b4b6df1d1be54208d4fdedfb407",
+                "sha256:d935946791e338f748e05a23df877d74fbcd39dc7b537f0aa2e5a5841cf7dde8",
+                "sha256:da92a1483a66cf2baa96de133c75e7d5d9a60e4a0e60d228f26c573c73a1feb6",
+                "sha256:efb8b41360c7e3d7700c147b70688aed0a03e86fbe5bcfdee079b0e634f026f9"
+            ],
+            "index": "pypi",
+            "version": "==0.1.3"
         },
         "hjson": {
             "hashes": [
@@ -836,26 +846,26 @@
         },
         "jsonschema": {
             "hashes": [
-                "sha256:8caf5b57a990a98e9b39832ef3cb35c176fe331414252b6e1b26fd5866f891a4",
-                "sha256:b508dd6142bd03f4c3670534c80af68cd7bbff9ea830b9cf2625d4a3c49ddf60"
+                "sha256:ce71d2f8c7983ef75a756e568317bf54bc531dc3ad7e66a128eae0d51623d8a3",
+                "sha256:dc274409c36175aad949c68e5ead0853aaffbe8e88c830ae66bb3c7a1728ad2d"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.18.0"
+            "version": "==4.18.6"
         },
         "jsonschema-specifications": {
             "hashes": [
-                "sha256:3d2b82663aff01815f744bb5c7887e2121a63399b49b104a3c96145474d091d7",
-                "sha256:ca1c4dd059a9e7b34101cf5b3ab7ff1d18b139f35950d598d629837ef66e8f28"
+                "sha256:05adf340b659828a004220a9613be00fa3f223f2b82002e273dee62fd50524b1",
+                "sha256:c91a50404e88a1f6ba40636778e2ee08f6e24c5613fe4c53ac24578a5a7f72bb"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2023.6.1"
+            "version": "==2023.7.1"
         },
         "kaggle": {
             "hashes": [
-                "sha256:fa29950f04bf6712e76e9f6fea5b9a3ee03bc971f63abb6a3df395d89a691d02"
+                "sha256:a6b3542f590cf341a5ad441d5ae021be93bd64441c96c758b154903495a3a600"
             ],
             "index": "pypi",
-            "version": "==1.5.15"
+            "version": "==1.5.16"
         },
         "lit": {
             "hashes": [
@@ -1124,11 +1134,11 @@
         },
         "neptune": {
             "hashes": [
-                "sha256:8e2d08a9282696fc6d1c52f3309d79daae366a8f9dc71c0f9d10177376d30d78",
-                "sha256:f4ea30fa6312659c86718e5116f71b9611372b741092a50c74422ba4db161550"
+                "sha256:be5ea84ea10309d7a44b5a0b899f08319a1ac1ee727460efb9c549ee36e4584d",
+                "sha256:d78cf6a74e23b679207e2ea64de72a720a5b12a007059c7277c6be6d5cc283b7"
             ],
             "index": "pypi",
-            "version": "==1.3.2"
+            "version": "==1.3.1"
         },
         "networkx": {
             "hashes": [
@@ -1162,34 +1172,34 @@
         },
         "numpy": {
             "hashes": [
-                "sha256:0ac6edfb35d2a99aaf102b509c8e9319c499ebd4978df4971b94419a116d0790",
-                "sha256:26815c6c8498dc49d81faa76d61078c4f9f0859ce7817919021b9eba72b425e3",
-                "sha256:4aedd08f15d3045a4e9c648f1e04daca2ab1044256959f1f95aafeeb3d794c16",
-                "sha256:4c69fe5f05eea336b7a740e114dec995e2f927003c30702d896892403df6dbf0",
-                "sha256:5177310ac2e63d6603f659fadc1e7bab33dd5a8db4e0596df34214eeab0fee3b",
-                "sha256:5aa48bebfb41f93043a796128854b84407d4df730d3fb6e5dc36402f5cd594c0",
-                "sha256:5b1b90860bf7d8a8c313b372d4f27343a54f415b20fb69dd601b7efe1029c91e",
-                "sha256:6c284907e37f5e04d2412950960894b143a648dea3f79290757eb878b91acbd1",
-                "sha256:6d183b5c58513f74225c376643234c369468e02947b47942eacbb23c1671f25d",
-                "sha256:7412125b4f18aeddca2ecd7219ea2d2708f697943e6f624be41aa5f8a9852cc4",
-                "sha256:7cd981ccc0afe49b9883f14761bb57c964df71124dcd155b0cba2b591f0d64b9",
-                "sha256:85cdae87d8c136fd4da4dad1e48064d700f63e923d5af6c8c782ac0df8044542",
-                "sha256:8aa130c3042052d656751df5e81f6d61edff3e289b5994edcf77f54118a8d9f4",
-                "sha256:95367ccd88c07af21b379be1725b5322362bb83679d36691f124a16357390153",
-                "sha256:9c7211d7920b97aeca7b3773a6783492b5b93baba39e7c36054f6e749fc7490c",
-                "sha256:9e3f2b96e3b63c978bc29daaa3700c028fe3f049ea3031b58aa33fe2a5809d24",
-                "sha256:b76aa836a952059d70a2788a2d98cb2a533ccd46222558b6970348939e55fc24",
-                "sha256:b792164e539d99d93e4e5e09ae10f8cbe5466de7d759fc155e075237e0c274e4",
-                "sha256:c0dc071017bc00abb7d7201bac06fa80333c6314477b3d10b52b58fa6a6e38f6",
-                "sha256:cc3fda2b36482891db1060f00f881c77f9423eead4c3579629940a3e12095fe8",
-                "sha256:d6b267f349a99d3908b56645eebf340cb58f01bd1e773b4eea1a905b3f0e4208",
-                "sha256:d76a84998c51b8b68b40448ddd02bd1081bb33abcdc28beee6cd284fe11036c6",
-                "sha256:e559c6afbca484072a98a51b6fa466aae785cfe89b69e8b856c3191bc8872a82",
-                "sha256:ecc68f11404930e9c7ecfc937aa423e1e50158317bf67ca91736a9864eae0232",
-                "sha256:f1accae9a28dc3cda46a91de86acf69de0d1b5f4edd44a9b0c3ceb8036dfff19"
+                "sha256:0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2",
+                "sha256:1a1329e26f46230bf77b02cc19e900db9b52f398d6722ca853349a782d4cff55",
+                "sha256:1b9735c27cea5d995496f46a8b1cd7b408b3f34b6d50459d9ac8fe3a20cc17bf",
+                "sha256:2792d23d62ec51e50ce4d4b7d73de8f67a2fd3ea710dcbc8563a51a03fb07b01",
+                "sha256:3e0746410e73384e70d286f93abf2520035250aad8c5714240b0492a7302fdca",
+                "sha256:4c3abc71e8b6edba80a01a52e66d83c5d14433cbcd26a40c329ec7ed09f37901",
+                "sha256:5883c06bb92f2e6c8181df7b39971a5fb436288db58b5a1c3967702d4278691d",
+                "sha256:5c97325a0ba6f9d041feb9390924614b60b99209a71a69c876f71052521d42a4",
+                "sha256:60e7f0f7f6d0eee8364b9a6304c2845b9c491ac706048c7e8cf47b83123b8dbf",
+                "sha256:76b4115d42a7dfc5d485d358728cdd8719be33cc5ec6ec08632a5d6fca2ed380",
+                "sha256:7dc869c0c75988e1c693d0e2d5b26034644399dd929bc049db55395b1379e044",
+                "sha256:834b386f2b8210dca38c71a6e0f4fd6922f7d3fcff935dbe3a570945acb1b545",
+                "sha256:8b77775f4b7df768967a7c8b3567e309f617dd5e99aeb886fa14dc1a0791141f",
+                "sha256:90319e4f002795ccfc9050110bbbaa16c944b1c37c0baeea43c5fb881693ae1f",
+                "sha256:b79e513d7aac42ae918db3ad1341a015488530d0bb2a6abcbdd10a3a829ccfd3",
+                "sha256:bb33d5a1cf360304754913a350edda36d5b8c5331a8237268c48f91253c3a364",
+                "sha256:bec1e7213c7cb00d67093247f8c4db156fd03075f49876957dca4711306d39c9",
+                "sha256:c5462d19336db4560041517dbb7759c21d181a67cb01b36ca109b2ae37d32418",
+                "sha256:c5652ea24d33585ea39eb6a6a15dac87a1206a692719ff45d53c5282e66d4a8f",
+                "sha256:d7806500e4f5bdd04095e849265e55de20d8cc4b661b038957354327f6d9b295",
+                "sha256:db3ccc4e37a6873045580d413fe79b68e47a681af8db2e046f1dacfa11f86eb3",
+                "sha256:dfe4a913e29b418d096e696ddd422d8a5d13ffba4ea91f9f60440a3b759b0187",
+                "sha256:eb942bfb6f84df5ce05dbf4b46673ffed0d3da59f13635ea9b926af3deb76926",
+                "sha256:f08f2e037bba04e707eebf4bc934f1972a315c883a9e0ebfa8a7756eabf9e357",
+                "sha256:fd608e19c8d7c55021dffd43bfe5492fab8cc105cc8986f813f8c3c048b38760"
             ],
             "index": "pypi",
-            "version": "==1.25.0"
+            "version": "==1.25.2"
         },
         "oauthlib": {
             "hashes": [
@@ -1249,14 +1259,19 @@
             "version": "==1.5.3"
         },
         "peft": {
-            "git": "https://github.com/huggingface/peft",
-            "ref": "39ef2546d5d9b8f5f8a7016ec10657887a867041"
+            "hashes": [
+                "sha256:2cf992772a6d703814477e0bdcdadd68cb8ea388111ce2d793dd2ff0e438f357",
+                "sha256:e768fa22d6e9f32aa7e891f0d06f355960278ca4dc0cdd96bff71f6f06269207"
+            ],
+            "index": "pypi",
+            "version": "==0.4.0"
         },
         "pillow": {
             "hashes": [
                 "sha256:00e65f5e822decd501e374b0650146063fbb30a7264b4d2744bdd7b913e0cab5",
                 "sha256:040586f7d37b34547153fa383f7f9aed68b738992380ac911447bb78f2abe530",
                 "sha256:0b6eb5502f45a60a3f411c63187db83a3d3107887ad0d036c13ce836f8a36f1d",
+                "sha256:1ce91b6ec08d866b14413d3f0bbdea7e24dfdc8e59f562bb77bc3fe60b6144ca",
                 "sha256:1f62406a884ae75fb2f818694469519fb685cc7eaff05d3451a9ebe55c646891",
                 "sha256:22c10cc517668d44b211717fd9775799ccec4124b9a7f7b3635fc5386e584992",
                 "sha256:3400aae60685b06bb96f99a21e1ada7bc7a413d5f49bce739828ecd9391bb8f7",
@@ -1286,6 +1301,7 @@
                 "sha256:9fb218c8a12e51d7ead2a7c9e101a04982237d4855716af2e9499306728fb485",
                 "sha256:a74ba0c356aaa3bb8e3eb79606a87669e7ec6444be352870623025d75a14a2bf",
                 "sha256:b4f69b3700201b80bb82c3a97d5e9254084f6dd5fb5b16fc1a7b974260f89f43",
+                "sha256:bc2ec7c7b5d66b8ec9ce9f720dbb5fa4bace0f545acd34870eff4a369b44bf37",
                 "sha256:c189af0545965fa8d3b9613cfdb0cd37f9d71349e0f7750e1fd704648d475ed2",
                 "sha256:c1fbe7621c167ecaa38ad29643d77a9ce7311583761abf7836e1510c580bf3dd",
                 "sha256:c7cf14a27b0d6adfaebb3ae4153f1e516df54e47e42dcc073d7b3d76111a8d86",
@@ -1312,6 +1328,14 @@
             "markers": "python_version >= '3.8'",
             "version": "==10.0.0"
         },
+        "pipreqs": {
+            "hashes": [
+                "sha256:a17f167880b6921be37533ce4c81ddc6e22b465c107aad557db43b1add56a99b",
+                "sha256:e522b9ed54aa3e8b7978ff251ab7a9af2f75d2cd8de4c102e881b666a79a308e"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.4.13"
+        },
         "portalocker": {
             "hashes": [
                 "sha256:032e81d534a88ec1736d03f780ba073f047a06c478b06e2937486f334e955c51",
@@ -1413,53 +1437,53 @@
         },
         "pydantic": {
             "hashes": [
-                "sha256:008c5e266c8aada206d0627a011504e14268a62091450210eda7c07fabe6963e",
-                "sha256:0588788a9a85f3e5e9ebca14211a496409cb3deca5b6971ff37c556d581854e7",
-                "sha256:08a6c32e1c3809fbc49debb96bf833164f3438b3696abf0fbeceb417d123e6eb",
-                "sha256:16928fdc9cb273c6af00d9d5045434c39afba5f42325fb990add2c241402d151",
-                "sha256:174899023337b9fc685ac8adaa7b047050616136ccd30e9070627c1aaab53a13",
-                "sha256:192c608ad002a748e4a0bed2ddbcd98f9b56df50a7c24d9a931a8c5dd053bd3d",
-                "sha256:1954f8778489a04b245a1e7b8b22a9d3ea8ef49337285693cf6959e4b757535e",
-                "sha256:2417de68290434461a266271fc57274a138510dca19982336639484c73a07af6",
-                "sha256:265a60da42f9f27e0b1014eab8acd3e53bd0bad5c5b4884e98a55f8f596b2c19",
-                "sha256:331c031ba1554b974c98679bd0780d89670d6fd6f53f5d70b10bdc9addee1713",
-                "sha256:373c0840f5c2b5b1ccadd9286782852b901055998136287828731868027a724f",
-                "sha256:3f34739a89260dfa420aa3cbd069fbcc794b25bbe5c0a214f8fb29e363484b66",
-                "sha256:41e0bb6efe86281623abbeeb0be64eab740c865388ee934cd3e6a358784aca6e",
-                "sha256:4400015f15c9b464c9db2d5d951b6a780102cfa5870f2c036d37c23b56f7fc1b",
-                "sha256:44e51ba599c3ef227e168424e220cd3e544288c57829520dc90ea9cb190c3248",
-                "sha256:469adf96c8e2c2bbfa655fc7735a2a82f4c543d9fee97bd113a7fb509bf5e622",
-                "sha256:5b02d24f7b2b365fed586ed73582c20f353a4c50e4be9ba2c57ab96f8091ddae",
-                "sha256:7522a7666157aa22b812ce14c827574ddccc94f361237ca6ea8bb0d5c38f1629",
-                "sha256:787cf23e5a0cde753f2eabac1b2e73ae3844eb873fd1f5bdbff3048d8dbb7604",
-                "sha256:8268a735a14c308923e8958363e3a3404f6834bb98c11f5ab43251a4e410170c",
-                "sha256:8dc77064471780262b6a68fe67e013298d130414d5aaf9b562c33987dbd2cf4f",
-                "sha256:a451ccab49971af043ec4e0d207cbc8cbe53dbf148ef9f19599024076fe9c25b",
-                "sha256:a6c098d4ab5e2d5b3984d3cb2527e2d6099d3de85630c8934efcfdc348a9760e",
-                "sha256:abade85268cc92dff86d6effcd917893130f0ff516f3d637f50dadc22ae93999",
-                "sha256:bc64eab9b19cd794a380179ac0e6752335e9555d214cfcb755820333c0784cb3",
-                "sha256:c3339a46bbe6013ef7bdd2844679bfe500347ac5742cd4019a88312aa58a9847",
-                "sha256:d185819a7a059550ecb85d5134e7d40f2565f3dd94cfd870132c5f91a89cf58c",
-                "sha256:d7781f1d13b19700b7949c5a639c764a077cbbdd4322ed505b449d3ca8edcb36",
-                "sha256:e297897eb4bebde985f72a46a7552a7556a3dd11e7f76acda0c1093e3dbcf216",
-                "sha256:e6cbfbd010b14c8a905a7b10f9fe090068d1744d46f9e0c021db28daeb8b6de1",
-                "sha256:e9738b0f2e6c70f44ee0de53f2089d6002b10c33264abee07bdb5c7f03038303",
-                "sha256:e9baf78b31da2dc3d3f346ef18e58ec5f12f5aaa17ac517e2ffd026a92a87588",
-                "sha256:ef55392ec4bb5721f4ded1096241e4b7151ba6d50a50a80a2526c854f42e6a2f",
-                "sha256:f66d479cf7eb331372c470614be6511eae96f1f120344c25f3f9bb59fb1b5528",
-                "sha256:fe429898f2c9dd209bd0632a606bddc06f8bce081bbd03d1c775a45886e2c1cb",
-                "sha256:ff44c5e89315b15ff1f7fdaf9853770b810936d6b01a7bcecaa227d2f8fe444f"
+                "sha256:0fe8a415cea8f340e7a9af9c54fc71a649b43e8ca3cc732986116b3cb135d303",
+                "sha256:1289c180abd4bd4555bb927c42ee42abc3aee02b0fb2d1223fb7c6e5bef87dbe",
+                "sha256:1eb2085c13bce1612da8537b2d90f549c8cbb05c67e8f22854e201bde5d98a47",
+                "sha256:2031de0967c279df0d8a1c72b4ffc411ecd06bac607a212892757db7462fc494",
+                "sha256:2a7bac939fa326db1ab741c9d7f44c565a1d1e80908b3797f7f81a4f86bc8d33",
+                "sha256:2d5a58feb9a39f481eda4d5ca220aa8b9d4f21a41274760b9bc66bfd72595b86",
+                "sha256:2f9a6fab5f82ada41d56b0602606a5506aab165ca54e52bc4545028382ef1c5d",
+                "sha256:2fcfb5296d7877af406ba1547dfde9943b1256d8928732267e2653c26938cd9c",
+                "sha256:549a8e3d81df0a85226963611950b12d2d334f214436a19537b2efed61b7639a",
+                "sha256:598da88dfa127b666852bef6d0d796573a8cf5009ffd62104094a4fe39599565",
+                "sha256:5d1197e462e0364906cbc19681605cb7c036f2475c899b6f296104ad42b9f5fb",
+                "sha256:69328e15cfda2c392da4e713443c7dbffa1505bc9d566e71e55abe14c97ddc62",
+                "sha256:6a9dfa722316f4acf4460afdf5d41d5246a80e249c7ff475c43a3a1e9d75cf62",
+                "sha256:6b30bcb8cbfccfcf02acb8f1a261143fab622831d9c0989707e0e659f77a18e0",
+                "sha256:6c076be61cd0177a8433c0adcb03475baf4ee91edf5a4e550161ad57fc90f523",
+                "sha256:771735dc43cf8383959dc9b90aa281f0b6092321ca98677c5fb6125a6f56d58d",
+                "sha256:795e34e6cc065f8f498c89b894a3c6da294a936ee71e644e4bd44de048af1405",
+                "sha256:87afda5539d5140cb8ba9e8b8c8865cb5b1463924d38490d73d3ccfd80896b3f",
+                "sha256:8fb2aa3ab3728d950bcc885a2e9eff6c8fc40bc0b7bb434e555c215491bcf48b",
+                "sha256:a1fcb59f2f355ec350073af41d927bf83a63b50e640f4dbaa01053a28b7a7718",
+                "sha256:a5e7add47a5b5a40c49b3036d464e3c7802f8ae0d1e66035ea16aa5b7a3923ed",
+                "sha256:a73f489aebd0c2121ed974054cb2759af8a9f747de120acd2c3394cf84176ccb",
+                "sha256:ab26038b8375581dc832a63c948f261ae0aa21f1d34c1293469f135fa92972a5",
+                "sha256:b0d191db0f92dfcb1dec210ca244fdae5cbe918c6050b342d619c09d31eea0cc",
+                "sha256:b749a43aa51e32839c9d71dc67eb1e4221bb04af1033a32e3923d46f9effa942",
+                "sha256:b7ccf02d7eb340b216ec33e53a3a629856afe1c6e0ef91d84a4e6f2fb2ca70fe",
+                "sha256:ba5b2e6fe6ca2b7e013398bc7d7b170e21cce322d266ffcd57cca313e54fb246",
+                "sha256:ba5c4a8552bff16c61882db58544116d021d0b31ee7c66958d14cf386a5b5350",
+                "sha256:c79e6a11a07da7374f46970410b41d5e266f7f38f6a17a9c4823db80dadf4303",
+                "sha256:ca48477862372ac3770969b9d75f1bf66131d386dba79506c46d75e6b48c1e09",
+                "sha256:dea7adcc33d5d105896401a1f37d56b47d443a2b2605ff8a969a0ed5543f7e33",
+                "sha256:e0a16d274b588767602b7646fa05af2782576a6cf1022f4ba74cbb4db66f6ca8",
+                "sha256:e4129b528c6baa99a429f97ce733fff478ec955513630e61b49804b6cf9b224a",
+                "sha256:e5f805d2d5d0a41633651a73fa4ecdd0b3d7a49de4ec3fadf062fe16501ddbf1",
+                "sha256:ef6c96b2baa2100ec91a4b428f80d8f28a3c9e53568219b6c298c1125572ebc6",
+                "sha256:fdbdd1d630195689f325c9ef1a12900524dceb503b00a987663ff4f58669b93d"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.10.11"
+            "version": "==1.10.12"
         },
         "pyjwt": {
             "hashes": [
-                "sha256:ba2b425b15ad5ef12f200dc67dd56af4e26de2331f965c5439994dad075876e1",
-                "sha256:bd6ca4a3c4285c1a2d4349e5a035fdf8fb94e04ccd0fcbe6ba289dae9cc3e074"
+                "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de",
+                "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.7.0"
+            "version": "==2.8.0"
         },
         "python-dateutil": {
             "hashes": [
@@ -1550,11 +1574,11 @@
         },
         "referencing": {
             "hashes": [
-                "sha256:90cb53782d550ba28d2166ef3f55731f38397def8832baac5d45235f1995e35e",
-                "sha256:d3c8f323ee1480095da44d55917cfb8278d73d6b4d5f677e3e40eb21314ac67f"
+                "sha256:47237742e990457f7512c7d27486394a9aadaf876cbfaa4be65b27b4f4d47c6b",
+                "sha256:c257b08a399b6c2f5a3510a50d28ab5dbc7bbde049bcaf954d43c446f83ab548"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.29.1"
+            "version": "==0.30.0"
         },
         "regex": {
             "hashes": [
@@ -1690,84 +1714,106 @@
         },
         "rpds-py": {
             "hashes": [
-                "sha256:0c62fdb01111da948e8446caaefebf2ca4307a58fcbc10039b48d0db7205397c",
-                "sha256:0df2ae878fd99342415a42659f3bcee34b12441a509033e0ab04c6e301895607",
-                "sha256:0dfec14a64759186153115d987f9e60b1fa7e8e14a00004a02482e696199e554",
-                "sha256:0e8da63b9baa154ec9ddd6dd397893830d17e5812ceb50edbae8122d8ecb9f2e",
-                "sha256:13aed64f2e0bef04a0eae6d1d1295f901f6c1640d1e20264dc3b19d62ef1721d",
-                "sha256:180963bb3e1fcc6ed6313ece5e065f0df4021a7eb7016084d3cbc90cd2a8af3e",
-                "sha256:1891903e567d728175c0475a1f0ffc1d1580013b0b265b9e2f1b8c93d58b2d05",
-                "sha256:18a97bb7f211b247346983092186927c517153ac155c611f43ca83d5ee93a3e2",
-                "sha256:1f60a7eb96fedcc5bf59761d33ac0f2f127d75f8c4b99ed0f138fc0f3601c537",
-                "sha256:22afed13c6ad4ccdc650ad44cbc06f835f4f30de201bd4ee2afc09bde06a357a",
-                "sha256:246a410e540ce7f635c6ad1b7aa00b7dcfc966c5f97217e41092c3f764dac4bf",
-                "sha256:247a1fd9bcdb373380cb8c0041417c9995fc163c9d88a39f5ec3d960114aca22",
-                "sha256:25a4b357ba7540d7cc709915699d67d505c8177cdb30824127092e1b4886b504",
-                "sha256:2d6a26953d3b291dd7e79e43bb203ed134ca889e63c8ebbc65e3ff98154303ef",
-                "sha256:300b8579740b06e246238b730e636f314a7d8dc475be1868650f5d3ddc29a0d8",
-                "sha256:316a66b0379a9e954872593aa2eb6d61949da2ecc8f572b4dafb07aa0e247171",
-                "sha256:358c7a976482537d26fcbdc7808e7cc25f64816fe89681f9aa8bef13e16c7370",
-                "sha256:36266e2e49b36ec6cc148e36667030d8e7f1c009edd2ca978ab09ed9c5a7589a",
-                "sha256:3660bd2afb23e1ca685df0d3092208fe6c7b2cae8c0be199b3da6d1a4bc91dc5",
-                "sha256:42eb3030665ee7a5c03fd4db6b8db1983aa91bcdffbed0f4687751deb2a94a7c",
-                "sha256:4333b5c2801e44bf342207a7700378a1013f300116c9553ce1ffbc73047d2a02",
-                "sha256:4b5164fdad37847e90683d3748dca7c4604f47ecd21e2d651dafc80393d1d314",
-                "sha256:4c66a5f9ca9c5fcb2527ad969541521f7216db713a7bd63aee52c3f5efa5924a",
-                "sha256:5b7c87503843a036f898d44c47326a117d23b6269a9f1574adf83d7bb051b839",
-                "sha256:657348b35a4c2e7c2340bf0bc37597900037bd87e9db7e6282711aaa77256e16",
-                "sha256:681ef7a21e6990583533c3a3038b7176f5e51e5d345fe2d9109b54f6bffcabcd",
-                "sha256:6b1b235b890373f785507f8f6a644a4f191b7195939e7f6108dc0e5e4fab57fd",
-                "sha256:6e85ea159f2d2132d4fcb71abb7df9683314f6073bf8ee9f9bd576440245e59c",
-                "sha256:6ee396f63a1f540fb3aecb8cc698180d30573a661be47fb3fff45cbd2b5d4686",
-                "sha256:7110854662ccf8db84b90e4624301ef5311cafff7e5f2a63f2d7cc0fc1a75b60",
-                "sha256:755a837fb7053dbf511fba26343423bd80d3b75a5d7f57f6e407c2fe5ae46682",
-                "sha256:7628b2080538faa4a1243b0002678cae7111af68ae7b5aa6cd8526762cace868",
-                "sha256:78c5577f99d2edc9eed9ec39fae27b73d04d1b2462aff6f6b11207e0364fc40d",
-                "sha256:79ac819182795a2168ed11075c7362de368f360244fb7cea8274c222b2d55365",
-                "sha256:7bd8dbc1c63668124e5e48e601d32f1053cfd5a86004ae0e55dc9ba8b1e7de29",
-                "sha256:87d74c2526115daa9d805a66377997602185a837ff7ecceed9d27e625c383572",
-                "sha256:8ac1e47ee4cb2dbd714537e63c67086eec63f56b13208fe450ae5be4f3d65671",
-                "sha256:8e1251d6690f356a305192089017da83999cded1d7e2405660d14c1dff976af7",
-                "sha256:8f64ef5700ff8fded62b12d1ea55463031cc5a353b670ed7146126c6cbf28788",
-                "sha256:961828c668140796c4963edb14cd968666d5414b9b5829997a4f475fd917906e",
-                "sha256:9af40bb89e40932e04c0cc1fb516249d6b3ae68ceebd984fdc592a6673244e50",
-                "sha256:9b331fa451d725258c1ad0ae6816cf36d55294e5cb68338cf91550b9a448a48f",
-                "sha256:a747477838a90b9264f434b5399309f9badb32c80ff3e9c4f6d5b87fddcbaa09",
-                "sha256:ac596301c7723809ecb6ec4cb2564b2cadc06f8c07b6348a56fcbf1ae043d751",
-                "sha256:b189640d59afa8aeff59865fa9d314bca97987c378950f215297e15d64ae1124",
-                "sha256:b3ed0d3498b69159db0d5db1393c8bae4df51cf936b2ef5cda53d800acab0019",
-                "sha256:b5a55fd01f61df19e4f53fd67b63ca9bf47559632c3f7d4037faa06d3a6fed54",
-                "sha256:b5cbcbd6451a4af2048fc0b21c15981462f6a378cb039aa53612c6e39958064e",
-                "sha256:b949e86affe17c8828d82936c51d7aa9b686511f9ac99a4b1de596d7140c8083",
-                "sha256:bfe98ad05400e7ac7a1df791645db08c66adc92d7a6449a406111303a265b71a",
-                "sha256:c0fe018430e5e8e1d8b513bcbccdb0ea34b9fd81c32b3a49c41a109fade17cfb",
-                "sha256:c26b1a0425c038cc23cf5770a47d7a7382fe68d6d10fb2a52c2896ca94e72550",
-                "sha256:c7019b2522af6b835118177b6b53f7ed08da28061aa5d44e06286be09799e7a4",
-                "sha256:c97e4cbe722474d17c309ee49e09331ceffe345bb72e2b10c6c740788871b122",
-                "sha256:cafbfa8f3a27e592bdcc420497e0c9a957c9f812b6c6ebfb7f961409215ba82d",
-                "sha256:cea42c2f52e37877e6b877bce64d109a6b0213e32545ecc70d4492d2a4641b8f",
-                "sha256:d2f8666fde7cfd9cdbc6c223876b39697d387f0215d12ed25147e9efec2dff5a",
-                "sha256:d418d2fd8c0fffe2897bc3e15a2e2ec87abf29076f0c36898cc33fb7881c2cbe",
-                "sha256:d4334405f6c73c29ff94521f78ad53ebb76a9c1b8dafea75852f9f64c3679cf7",
-                "sha256:d6696c2a002e982e89975063939a2579623d6d6b24634b147848ec9f35dad7d6",
-                "sha256:d7e46f52272ceecc42c05ad869b068b2dbfb6eb5643bcccecd2327d3cded5a2e",
-                "sha256:db71c665fc7ddb9ac53d7b69dc588493c0b71635b28fc8ff64b31eb9db5b3461",
-                "sha256:ddb5226b11f4fce98c6e47c07819fbbfdda6d3fd529cd176ad8a1e0b98a70b05",
-                "sha256:df3097abf5fd09bfcd8f6fd02d052b25cc3e160b3ee71b6fbd831b2cd516c958",
-                "sha256:e086610118163400a1822af0ee857581c0e047aa50a9c3543d17fbe65183a339",
-                "sha256:e1700fba17dd63c9c7d5cb03cf838db23cf938dd5cdb1eecb6ba8b8da62c3e73",
-                "sha256:e261fa453ad50fe1d9287fa21d962cdbcd3d495cf1160dd0f893883040c455b6",
-                "sha256:e4e11f71673905d9e8735b8dd4ef8fa85a82e6003851fe46f9bdc51aebc2cd5d",
-                "sha256:ea8c4a1232c7bacf73366b0490dda7f67d25958aec1c2f099b6753c8bfe84427",
-                "sha256:ed823997c3300b541da0fcc9eee8fbe6740e07939ffa432824cfca287472d652",
-                "sha256:ee42ce4ef46ea334ce8ab63d5a57c7fd78238c9c7293b3caa6dfedf11bd28773",
-                "sha256:f1c84912d77b01651488bbe392df593b4c5852e213477e268ebbb7c799059d78",
-                "sha256:f81a570a20f9fce617d728f4e3bdc05bfbb68afa2e795ec0c52544a7923de517",
-                "sha256:f8a1cc37e2395002381510a6862c29634acd67edfe5774661a6c48d8617acae7",
-                "sha256:fffd98c1fd6b38df35e471549c2486d826af0fda6ca55c0bbbb956b291e7f2ae"
+                "sha256:0173c0444bec0a3d7d848eaeca2d8bd32a1b43f3d3fde6617aac3731fa4be05f",
+                "sha256:01899794b654e616c8625b194ddd1e5b51ef5b60ed61baa7a2d9c2ad7b2a4238",
+                "sha256:02938432352359805b6da099c9c95c8a0547fe4b274ce8f1a91677401bb9a45f",
+                "sha256:03421628f0dc10a4119d714a17f646e2837126a25ac7a256bdf7c3943400f67f",
+                "sha256:03975db5f103997904c37e804e5f340c8fdabbb5883f26ee50a255d664eed58c",
+                "sha256:0766babfcf941db8607bdaf82569ec38107dbb03c7f0b72604a0b346b6eb3298",
+                "sha256:07e2c54bef6838fa44c48dfbc8234e8e2466d851124b551fc4e07a1cfeb37260",
+                "sha256:0836d71ca19071090d524739420a61580f3f894618d10b666cf3d9a1688355b1",
+                "sha256:095b460e117685867d45548fbd8598a8d9999227e9061ee7f012d9d264e6048d",
+                "sha256:0e7521f5af0233e89939ad626b15278c71b69dc1dfccaa7b97bd4cdf96536bb7",
+                "sha256:0f2996fbac8e0b77fd67102becb9229986396e051f33dbceada3debaacc7033f",
+                "sha256:1054a08e818f8e18910f1bee731583fe8f899b0a0a5044c6e680ceea34f93876",
+                "sha256:13b602dc3e8dff3063734f02dcf05111e887f301fdda74151a93dbbc249930fe",
+                "sha256:141acb9d4ccc04e704e5992d35472f78c35af047fa0cfae2923835d153f091be",
+                "sha256:14c408e9d1a80dcb45c05a5149e5961aadb912fff42ca1dd9b68c0044904eb32",
+                "sha256:159fba751a1e6b1c69244e23ba6c28f879a8758a3e992ed056d86d74a194a0f3",
+                "sha256:190ca6f55042ea4649ed19c9093a9be9d63cd8a97880106747d7147f88a49d18",
+                "sha256:196cb208825a8b9c8fc360dc0f87993b8b260038615230242bf18ec84447c08d",
+                "sha256:1fcdee18fea97238ed17ab6478c66b2095e4ae7177e35fb71fbe561a27adf620",
+                "sha256:207f57c402d1f8712618f737356e4b6f35253b6d20a324d9a47cb9f38ee43a6b",
+                "sha256:24a81c177379300220e907e9b864107614b144f6c2a15ed5c3450e19cf536fae",
+                "sha256:29cd8bfb2d716366a035913ced99188a79b623a3512292963d84d3e06e63b496",
+                "sha256:2d8b3b3a2ce0eaa00c5bbbb60b6713e94e7e0becab7b3db6c5c77f979e8ed1f1",
+                "sha256:35da5cc5cb37c04c4ee03128ad59b8c3941a1e5cd398d78c37f716f32a9b7f67",
+                "sha256:44659b1f326214950a8204a248ca6199535e73a694be8d3e0e869f820767f12f",
+                "sha256:47c5f58a8e0c2c920cc7783113df2fc4ff12bf3a411d985012f145e9242a2764",
+                "sha256:4bd4dc3602370679c2dfb818d9c97b1137d4dd412230cfecd3c66a1bf388a196",
+                "sha256:4ea6b73c22d8182dff91155af018b11aac9ff7eca085750455c5990cb1cfae6e",
+                "sha256:50025635ba8b629a86d9d5474e650da304cb46bbb4d18690532dd79341467846",
+                "sha256:517cbf6e67ae3623c5127206489d69eb2bdb27239a3c3cc559350ef52a3bbf0b",
+                "sha256:5855c85eb8b8a968a74dc7fb014c9166a05e7e7a8377fb91d78512900aadd13d",
+                "sha256:5a46859d7f947061b4010e554ccd1791467d1b1759f2dc2ec9055fa239f1bc26",
+                "sha256:65a0583c43d9f22cb2130c7b110e695fff834fd5e832a776a107197e59a1898e",
+                "sha256:674c704605092e3ebbbd13687b09c9f78c362a4bc710343efe37a91457123044",
+                "sha256:682726178138ea45a0766907957b60f3a1bf3acdf212436be9733f28b6c5af3c",
+                "sha256:686ba516e02db6d6f8c279d1641f7067ebb5dc58b1d0536c4aaebb7bf01cdc5d",
+                "sha256:6a5d3fbd02efd9cf6a8ffc2f17b53a33542f6b154e88dd7b42ef4a4c0700fdad",
+                "sha256:6aa8326a4a608e1c28da191edd7c924dff445251b94653988efb059b16577a4d",
+                "sha256:700375326ed641f3d9d32060a91513ad668bcb7e2cffb18415c399acb25de2ab",
+                "sha256:71f2f7715935a61fa3e4ae91d91b67e571aeb5cb5d10331ab681256bda2ad920",
+                "sha256:745f5a43fdd7d6d25a53ab1a99979e7f8ea419dfefebcab0a5a1e9095490ee5e",
+                "sha256:79f594919d2c1a0cc17d1988a6adaf9a2f000d2e1048f71f298b056b1018e872",
+                "sha256:7d68dc8acded354c972116f59b5eb2e5864432948e098c19fe6994926d8e15c3",
+                "sha256:7f67da97f5b9eac838b6980fc6da268622e91f8960e083a34533ca710bec8611",
+                "sha256:83b32f0940adec65099f3b1c215ef7f1d025d13ff947975a055989cb7fd019a4",
+                "sha256:876bf9ed62323bc7dcfc261dbc5572c996ef26fe6406b0ff985cbcf460fc8a4c",
+                "sha256:890ba852c16ace6ed9f90e8670f2c1c178d96510a21b06d2fa12d8783a905193",
+                "sha256:8b08605d248b974eb02f40bdcd1a35d3924c83a2a5e8f5d0fa5af852c4d960af",
+                "sha256:8b2eb034c94b0b96d5eddb290b7b5198460e2d5d0c421751713953a9c4e47d10",
+                "sha256:8b9ec12ad5f0a4625db34db7e0005be2632c1013b253a4a60e8302ad4d462afd",
+                "sha256:8c8d7594e38cf98d8a7df25b440f684b510cf4627fe038c297a87496d10a174f",
+                "sha256:8d3335c03100a073883857e91db9f2e0ef8a1cf42dc0369cbb9151c149dbbc1b",
+                "sha256:8d70e8f14900f2657c249ea4def963bed86a29b81f81f5b76b5a9215680de945",
+                "sha256:9039a11bca3c41be5a58282ed81ae422fa680409022b996032a43badef2a3752",
+                "sha256:91378d9f4151adc223d584489591dbb79f78814c0734a7c3bfa9c9e09978121c",
+                "sha256:9251eb8aa82e6cf88510530b29eef4fac825a2b709baf5b94a6094894f252387",
+                "sha256:933a7d5cd4b84f959aedeb84f2030f0a01d63ae6cf256629af3081cf3e3426e8",
+                "sha256:978fa96dbb005d599ec4fd9ed301b1cc45f1a8f7982d4793faf20b404b56677d",
+                "sha256:987b06d1cdb28f88a42e4fb8a87f094e43f3c435ed8e486533aea0bf2e53d931",
+                "sha256:99b1c16f732b3a9971406fbfe18468592c5a3529585a45a35adbc1389a529a03",
+                "sha256:99e7c4bb27ff1aab90dcc3e9d37ee5af0231ed98d99cb6f5250de28889a3d502",
+                "sha256:9c439fd54b2b9053717cca3de9583be6584b384d88d045f97d409f0ca867d80f",
+                "sha256:9ea4d00850ef1e917815e59b078ecb338f6a8efda23369677c54a5825dbebb55",
+                "sha256:9f30d205755566a25f2ae0382944fcae2f350500ae4df4e795efa9e850821d82",
+                "sha256:a06418fe1155e72e16dddc68bb3780ae44cebb2912fbd8bb6ff9161de56e1798",
+                "sha256:a0805911caedfe2736935250be5008b261f10a729a303f676d3d5fea6900c96a",
+                "sha256:a1f044792e1adcea82468a72310c66a7f08728d72a244730d14880cd1dabe36b",
+                "sha256:a216b26e5af0a8e265d4efd65d3bcec5fba6b26909014effe20cd302fd1138fa",
+                "sha256:a987578ac5214f18b99d1f2a3851cba5b09f4a689818a106c23dbad0dfeb760f",
+                "sha256:aad51239bee6bff6823bbbdc8ad85136c6125542bbc609e035ab98ca1e32a192",
+                "sha256:ab2299e3f92aa5417d5e16bb45bb4586171c1327568f638e8453c9f8d9e0f020",
+                "sha256:ab6919a09c055c9b092798ce18c6c4adf49d24d4d9e43a92b257e3f2548231e7",
+                "sha256:b0c43f8ae8f6be1d605b0465671124aa8d6a0e40f1fb81dcea28b7e3d87ca1e1",
+                "sha256:b1440c291db3f98a914e1afd9d6541e8fc60b4c3aab1a9008d03da4651e67386",
+                "sha256:b52e7c5ae35b00566d244ffefba0f46bb6bec749a50412acf42b1c3f402e2c90",
+                "sha256:bf4151acb541b6e895354f6ff9ac06995ad9e4175cbc6d30aaed08856558201f",
+                "sha256:c27ee01a6c3223025f4badd533bea5e87c988cb0ba2811b690395dfe16088cfe",
+                "sha256:c545d9d14d47be716495076b659db179206e3fd997769bc01e2d550eeb685596",
+                "sha256:c5934e2833afeaf36bd1eadb57256239785f5af0220ed8d21c2896ec4d3a765f",
+                "sha256:c7671d45530fcb6d5e22fd40c97e1e1e01965fc298cbda523bb640f3d923b387",
+                "sha256:c861a7e4aef15ff91233751619ce3a3d2b9e5877e0fcd76f9ea4f6847183aa16",
+                "sha256:d25b1c1096ef0447355f7293fbe9ad740f7c47ae032c2884113f8e87660d8f6e",
+                "sha256:d55777a80f78dd09410bd84ff8c95ee05519f41113b2df90a69622f5540c4f8b",
+                "sha256:d576c3ef8c7b2d560e301eb33891d1944d965a4d7a2eacb6332eee8a71827db6",
+                "sha256:dd9da77c6ec1f258387957b754f0df60766ac23ed698b61941ba9acccd3284d1",
+                "sha256:de0b6eceb46141984671802d412568d22c6bacc9b230174f9e55fc72ef4f57de",
+                "sha256:e07e5dbf8a83c66783a9fe2d4566968ea8c161199680e8ad38d53e075df5f0d0",
+                "sha256:e564d2238512c5ef5e9d79338ab77f1cbbda6c2d541ad41b2af445fb200385e3",
+                "sha256:ed89861ee8c8c47d6beb742a602f912b1bb64f598b1e2f3d758948721d44d468",
+                "sha256:ef1f08f2a924837e112cba2953e15aacfccbbfcd773b4b9b4723f8f2ddded08e",
+                "sha256:f411330a6376fb50e5b7a3e66894e4a39e60ca2e17dce258d53768fea06a37bd",
+                "sha256:f68996a3b3dc9335037f82754f9cdbe3a95db42bde571d8c3be26cc6245f2324",
+                "sha256:f7fdf55283ad38c33e35e2855565361f4bf0abd02470b8ab28d499c663bc5d7c",
+                "sha256:f963c6b1218b96db85fc37a9f0851eaf8b9040aa46dec112611697a7023da535",
+                "sha256:fa2818759aba55df50592ecbc95ebcdc99917fa7b55cc6796235b04193eb3c55",
+                "sha256:fae5cb554b604b3f9e2c608241b5d8d303e410d7dfb6d397c335f983495ce7f6",
+                "sha256:fb39aca7a64ad0c9490adfa719dbeeb87d13be137ca189d2564e596f8ba32c07"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.8.8"
+            "version": "==0.9.2"
         },
         "s3transfer": {
             "hashes": [
@@ -2172,11 +2218,11 @@
         },
         "threadpoolctl": {
             "hashes": [
-                "sha256:8b99adda265feb6773280df41eece7b2e6561b772d21ffd52e372f999024907b",
-                "sha256:a335baacfaa4400ae1f0d8e3a58d6674d2f8828e3716bb2802c44955ad391380"
+                "sha256:2b7818516e423bdaebb97c723f86a7c6b0a83d3f3b0970328d66f4d9104dc032",
+                "sha256:c96a0ba3bdddeaca37dc4cc7344aafad41cdb8c313f74fdfe387a867bba93355"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.1.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==3.2.0"
         },
         "tiktoken": {
             "hashes": [
@@ -2268,45 +2314,45 @@
         },
         "torch": {
             "hashes": [
-                "sha256:19e17e9ea7fb3e1cbd5c8585cf18300653216bcd27858825dc93b74e5495360a",
-                "sha256:64c176ebff6904155aa6f72b0f996c9ea17f29b8af7aa9afeee8bff726f91ef3",
-                "sha256:726cf790bf5730d89c05fe80c1c64f9cf02d09180da1891ee78ecd5891acadcd",
-                "sha256:a9f086988e659674ac85d4495f5fb5e6dc9a64b99746277e760792fa44010097",
-                "sha256:b078675648025f1dae1cdc8955f975f5ca81167809e9662b1481e456171ebfb9",
-                "sha256:c4dbc3f7f3eff6576473c3711d5d99adaaef733490b39de4970980d6edf4f0c2",
-                "sha256:f0b525686f25c30e1de87d0fbdcd0b373f4c70a0f72bd854389e601a52fdc5e5",
-                "sha256:f6e26492d214edab5b407e903ed3c7b190ac5709330bd72060d1be01b354c198"
+                "sha256:1f8efaebfcbb7ec3962fd8c7c3be02c6666eff53a12043006a749d647656163e",
+                "sha256:238573d362c564113451046f6708c3b8158fe6b1b7f6c03b7273327d955deb54",
+                "sha256:2588926725750c9ba799c133d4f6ee8fa477f6f0d88d6c2cebfe5bcfe8d7d7c3",
+                "sha256:4b690e2b77f21073500c65d8bb9ea9656b8cb4e969f357370bbc992a3b074764",
+                "sha256:5ee2b7c19265b9c869525c378fcdf350510b8f3fc08af26da1a2587a34cea8f5",
+                "sha256:893f6dd205316a104b04b69877bc40ab9908428274920094c17b81396a8c985c",
+                "sha256:dbba5b085722d24d617102954d9a9023ee4f7584148a9e465afb0c9696d15517",
+                "sha256:eab97a9fe59e7e31d6562b186f435e717b1df3331cadc776e6c0732239a9ed39"
             ],
             "index": "pytorch",
-            "version": "==2.0.0+cu117"
+            "version": "==2.0.0+cu118"
         },
         "torchaudio": {
             "hashes": [
-                "sha256:32a3490d9d748e8e8857a35d0206e0f0cb264622098f7b81257c35887fec8398",
-                "sha256:59ae4cea4011175e4e549f5d083830f96986a43783d920ff73059147cc0c3680",
-                "sha256:5f06e90216934e61cfe04786bb327ae07cb6c67f5fca6e0acf87ca4186ab1bbf",
-                "sha256:8006e44bc98fc48cece6b9e4c09245b2b06bcf0c6893214ed7de82dea6722e6d",
-                "sha256:ad38ec3724fb61898f3a71c592cf4dd9a60ed14ab1b15f7d5c7317f35516ec5f",
-                "sha256:be88ea47519debb00a1a0e0fe36fd1f6e5c25bffffc775ab2cecd3fe7379d0e6",
-                "sha256:e37a556d036f48a4f4f33fe698f844024037ece66123bf21715de73e506cdec3",
-                "sha256:f8fc84a567e307c8d05742ca713240c43cd67842632f6b231adf3fcffd812f37"
+                "sha256:070b3849696142be3d91e0b27076da1f3f8cb432abcce35cd956d191fc0d8187",
+                "sha256:19c4ef9012324c4fb80ea66934551b7807d97148c28538e2eabafe16ab50e91c",
+                "sha256:1a2722b6f66578edc563e77b4cc07a22d663a2bb5ee5c9de2539661a98a14db9",
+                "sha256:3e884a271e6a1eb67a8694042bccb9f25a71bcdfbf0037222ac034193a176ead",
+                "sha256:8acb163aa31e694b7bba96d659146d8cb10ea3faab8950b97406f3bcf0395005",
+                "sha256:8cdc67cf83b5c77432c78f5b6adcc8c2c69ddf98744543a2216d26d7ce614546",
+                "sha256:94b0e9c76ca91d1e0289895920fbfe8c117464d757b64b5dfadb1308cf9904f9",
+                "sha256:c7054eebe9f0f0ff74cb5bf883375f8164db6a89ed08007040cb336f6010375b"
             ],
             "index": "pytorch",
-            "version": "==2.0.1+cu117"
+            "version": "==2.0.1+cu118"
         },
         "torchvision": {
             "hashes": [
-                "sha256:3a6fbdeffe6a1a9488d3ddcb9f402b1b2ec29a6e59aa68acbd986e627a996ef4",
-                "sha256:3d013054909eda4f275fe02914e9e245539bcb9bfa395ea353e094cfc050398a",
-                "sha256:3fbfa46bc1ec932af1ca82f6cde1ca99304839322b6b06d97a99e3ba6fee4384",
-                "sha256:5c4416460c531a380a4e989ecf60a3bce1ccfff9d9c9e1eb51fa30de2b6d91ab",
-                "sha256:67ff110078913bf684893c4a13d9415f11f6908f40f33fa6626726913196b279",
-                "sha256:699905f365c12eac8d123e8a13cd967582aac04b63f07118c2459933682f9645",
-                "sha256:6d82ede144cabe85f21814c723b3a666d427de9566056701863663ea0f2ad7d9",
-                "sha256:a9bff1003aae1019f672996ebee62438f83843498926733d6f2f378314ad8326"
+                "sha256:22fe045677a74afecf7cd2f4146dca93dc926f40d0a658aa1bed3275f87dc7de",
+                "sha256:4008dee965103030a196d385ca8d84a9c20fc52585f5753fc8360219a7a06c4e",
+                "sha256:473ea5fba33c0e8fc482cf70ced986521f2a3d79497b0bdaa435353ca8752e5e",
+                "sha256:5ece279c8f521f8f6373bf971f2adc63a961d694c4aced538df81294210ee4ba",
+                "sha256:9104730d655abf282c2845d4cd472bb08936f3c85083a28aefd76ad9eeaff261",
+                "sha256:9a679fa37a741018c804234693bbac3d487fb3dd55ee73f6b33677b177c8c07a",
+                "sha256:9ced2c90ee782bbb415b0de65bcc10d4fe811131864679b707442c9d9e8aaafd",
+                "sha256:9ff719de1c09f1825492e3e39f4e49c42782e559298529e4b47d3f1404c45d1f"
             ],
             "index": "pytorch",
-            "version": "==0.15.1+cu117"
+            "version": "==0.15.1+cu118"
         },
         "tornado": {
             "hashes": [
@@ -2335,11 +2381,11 @@
         },
         "transformers": {
             "hashes": [
-                "sha256:9b12bd9d69f21b7c56cd512117fd52856b3def1c9bfc1da97ab0ee4e8bcbd797",
-                "sha256:fa74fc271d0692f385d571ce83ec898e3350455f6076d21631f4eed4916e6ffd"
+                "sha256:4302fba920a1c24d3a429a29efff6a63eac03f3f3cf55b55927fc795d01cb273",
+                "sha256:8487aab0195ce1c2a5ae189305118b9720daddbc7b688edb09ccd79e3b149f6b"
             ],
             "index": "pypi",
-            "version": "==4.30.1"
+            "version": "==4.31.0"
         },
         "triton": {
             "hashes": [
@@ -2390,11 +2436,11 @@
         },
         "uvicorn": {
             "hashes": [
-                "sha256:79277ae03db57ce7d9aa0567830bbb51d7a612f54d6e1e3e92da3ef24c2c8ed8",
-                "sha256:e9434d3bbf05f310e762147f769c9f21235ee118ba2d2bf1155a7196448bd996"
+                "sha256:1f9be6558f01239d4fdf22ef8126c39cb1ad0addf76c40e760549d2c2f43ab53",
+                "sha256:4d3cc12d7727ba72b64d12d3cc7743124074c0a69f7b201512fc50c3e3f1569a"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==0.22.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==0.23.2"
         },
         "wcwidth": {
             "hashes": [
@@ -2427,107 +2473,101 @@
         },
         "xxhash": {
             "hashes": [
-                "sha256:01f36b671ff55cb1d5c2f6058b799b697fd0ae4b4582bba6ed0999678068172a",
-                "sha256:02badf3754e2133de254a4688798c4d80f0060635087abcb461415cb3eb82115",
-                "sha256:052fd0efdd5525c2dbc61bebb423d92aa619c4905bba605afbf1e985a562a231",
-                "sha256:0a2cdfb5cae9fafb9f7b65fd52ecd60cf7d72c13bb2591ea59aaefa03d5a8827",
-                "sha256:0a6d58ba5865475e53d6c2c4fa6a62e2721e7875e146e2681e5337a6948f12e7",
-                "sha256:0d54ac023eef7e3ac9f0b8841ae8a376b933043bc2ad428121346c6fa61c491c",
-                "sha256:0dcb419bf7b0bc77d366e5005c25682249c5521a63fd36c51f584bd91bb13bd5",
-                "sha256:0eea848758e4823a01abdbcccb021a03c1ee4100411cbeeb7a5c36a202a0c13c",
-                "sha256:11bf87dc7bb8c3b0b5e24b7b941a9a19d8c1f88120b6a03a17264086bc8bb023",
-                "sha256:17b65454c5accbb079c45eca546c27c4782f5175aa320758fafac896b1549d27",
-                "sha256:1a42994f0d42b55514785356722d9031f064fd34e495b3a589e96db68ee0179d",
-                "sha256:1afb9b9d27fd675b436cb110c15979976d92d761ad6e66799b83756402f3a974",
-                "sha256:1afd47af8955c5db730f630ad53ae798cf7fae0acb64cebb3cf94d35c47dd088",
-                "sha256:1bdd57973e2b802ef32553d7bebf9402dac1557874dbe5c908b499ea917662cd",
-                "sha256:20181cbaed033c72cb881b2a1d13c629cd1228f113046133469c9a48cfcbcd36",
-                "sha256:2198c4901a0223c48f6ec0a978b60bca4f4f7229a11ca4dc96ca325dd6a29115",
-                "sha256:2408d49260b0a4a7cc6ba445aebf38e073aeaf482f8e32767ca477e32ccbbf9e",
-                "sha256:26cb52174a7e96a17acad27a3ca65b24713610ac479c99ac9640843822d3bebf",
-                "sha256:2783d41487ce6d379fdfaa7332fca5187bf7010b9bddcf20cafba923bc1dc665",
-                "sha256:3126df6520cbdbaddd87ce74794b2b6c45dd2cf6ac2b600a374b8cdb76a2548c",
-                "sha256:314ec0bd21f0ee8d30f2bd82ed3759314bd317ddbbd8555668f3d20ab7a8899a",
-                "sha256:368265392cb696dd53907e2328b5a8c1bee81cf2142d0cc743caf1c1047abb36",
-                "sha256:3a26eeb4625a6e61cedc8c1b39b89327c9c7e1a8c2c4d786fe3f178eb839ede6",
-                "sha256:3a68d1e8a390b660d94b9360ae5baa8c21a101bd9c4790a8b30781bada9f1fc6",
-                "sha256:3b1f3c6d67fa9f49c4ff6b25ce0e7143bab88a5bc0f4116dd290c92337d0ecc7",
-                "sha256:3d4b15c00e807b1d3d0b612338c814739dec310b80fb069bd732b98ddc709ad7",
-                "sha256:3f4152fd0bf8b03b79f2f900fd6087a66866537e94b5a11fd0fd99ef7efe5c42",
-                "sha256:498843b66b9ca416e9d03037e5875c8d0c0ab9037527e22df3b39aa5163214cd",
-                "sha256:49f51fab7b762da7c2cee0a3d575184d3b9be5e2f64f26cae2dd286258ac9b3c",
-                "sha256:4b948a03f89f5c72d69d40975af8af241111f0643228796558dc1cae8f5560b0",
-                "sha256:4ec1f57127879b419a2c8d2db9d9978eb26c61ae17e5972197830430ae78d25b",
-                "sha256:50ce82a71b22a3069c02e914bf842118a53065e2ec1c6fb54786e03608ab89cc",
-                "sha256:5384f1d9f30876f5d5b618464fb19ff7ce6c0fe4c690fbaafd1c52adc3aae807",
-                "sha256:561076ca0dcef2fbc20b2bc2765bff099e002e96041ae9dbe910a863ca6ee3ea",
-                "sha256:59dc8bfacf89b8f5be54d55bc3b4bd6d74d0c5320c8a63d2538ac7df5b96f1d5",
-                "sha256:5daff3fb5bfef30bc5a2cb143810d376d43461445aa17aece7210de52adbe151",
-                "sha256:61b0bcf946fdfd8ab5f09179dc2b5c74d1ef47cedfc6ed0ec01fdf0ee8682dd3",
-                "sha256:61e6aa1d30c2af692aa88c4dd48709426e8b37bff6a574ee2de677579c34a3d6",
-                "sha256:649cdf19df175925ad87289ead6f760cd840730ee85abc5eb43be326a0a24d97",
-                "sha256:66b8a90b28c13c2aae7a71b32638ceb14cefc2a1c8cf23d8d50dfb64dfac7aaf",
-                "sha256:6b7c9aa77bbce61a5e681bd39cb6a804338474dcc90abe3c543592aa5d6c9a9b",
-                "sha256:75aa692936942ccb2e8fd6a386c81c61630ac1b6d6e921698122db8a930579c3",
-                "sha256:75bb5be3c5de702a547715f320ecf5c8014aeca750ed5147ca75389bd22e7343",
-                "sha256:761df3c7e2c5270088b691c5a8121004f84318177da1ca1db64222ec83c44871",
-                "sha256:77709139af5123c578ab06cf999429cdb9ab211047acd0c787e098dcb3f1cb4d",
-                "sha256:7deae3a312feb5c17c97cbf18129f83cbd3f1f9ec25b0f50e2bd9697befb22e7",
-                "sha256:82daaab720866bf690b20b49de5640b0c27e3b8eea2d08aa75bdca2b0f0cfb63",
-                "sha256:883dc3d3942620f4c7dbc3fd6162f50a67f050b714e47da77444e3bcea7d91cc",
-                "sha256:89585adc73395a10306d2e2036e50d6c4ac0cf8dd47edf914c25488871b64f6d",
-                "sha256:8970f6a411a9839a02b23b7e90bbbba4a6de52ace009274998566dc43f36ca18",
-                "sha256:91687671fd9d484a4e201ad266d366b695a45a1f2b41be93d116ba60f1b8f3b3",
-                "sha256:919bc1b010aa6ff0eb918838ff73a435aed9e9a19c3202b91acecd296bf75607",
-                "sha256:92fd765591c83e5c5f409b33eac1d3266c03d3d11c71a7dbade36d5cdee4fbc0",
-                "sha256:994e4741d5ed70fc2a335a91ef79343c6b1089d7dfe6e955dd06f8ffe82bede6",
-                "sha256:9b94749130ef3119375c599bfce82142c2500ef9ed3280089157ee37662a7137",
-                "sha256:9d3f686e3d1c8900c5459eee02b60c7399e20ec5c6402364068a343c83a61d90",
-                "sha256:9eba0c7c12126b12f7fcbea5513f28c950d28f33d2a227f74b50b77789e478e8",
-                "sha256:a0e1bd0260c1da35c1883321ce2707ceea07127816ab625e1226ec95177b561a",
-                "sha256:a0f7a16138279d707db778a63264d1d6016ac13ffd3f1e99f54b2855d6c0d8e1",
-                "sha256:a32d546a1752e4ee7805d6db57944f7224afa7428d22867006b6486e4195c1f3",
-                "sha256:a433f6162b18d52f7068175d00bd5b1563b7405f926a48d888a97b90a160c40d",
-                "sha256:a892b4b139126a86bfdcb97cd912a2f8c4e8623869c3ef7b50871451dd7afeb0",
-                "sha256:a910b1193cd90af17228f5d6069816646df0148f14f53eefa6b2b11a1dedfcd0",
-                "sha256:aabdbc082030f8df613e2d2ea1f974e7ad36a539bdfc40d36f34e55c7e4b8e94",
-                "sha256:add774341c09853b1612c64a526032d95ab1683053325403e1afbe3ad2f374c5",
-                "sha256:ae521ed9287f86aac979eeac43af762f03d9d9797b2272185fb9ddd810391216",
-                "sha256:af44b9e59c4b2926a4e3c7f9d29949ff42fcea28637ff6b8182e654461932be8",
-                "sha256:b0c094d5e65a46dbf3fe0928ff20873a747e6abfd2ed4b675beeb2750624bc2e",
-                "sha256:b0d16775094423088ffa357d09fbbb9ab48d2fb721d42c0856b801c86f616eec",
-                "sha256:b5019fb33711c30e54e4e57ae0ca70af9d35b589d385ac04acd6954452fa73bb",
-                "sha256:baa99cebf95c1885db21e119395f222a706a2bb75a545f0672880a442137725e",
-                "sha256:bb6d8ce31dc25faf4da92991320e211fa7f42de010ef51937b1dc565a4926501",
-                "sha256:bbc30c98ab006ab9fc47e5ed439c00f706bc9d4441ff52693b8b6fea335163e0",
-                "sha256:c55fa832fc3fe64e0d29da5dc9b50ba66ca93312107cec2709300ea3d3bab5c7",
-                "sha256:c5e8db6e1ee7267b7c412ad0afd5863bf7a95286b8333a5958c8097c69f94cf5",
-                "sha256:c5f3e33fe6cbab481727f9aeb136a213aed7e33cd1ca27bd75e916ffacc18411",
-                "sha256:cc8878935671490efe9275fb4190a6062b73277bd273237179b9b5a2aa436153",
-                "sha256:ce7c3ce28f94302df95eaea7c9c1e2c974b6d15d78a0c82142a97939d7b6c082",
-                "sha256:cead7c0307977a00b3f784cff676e72c147adbcada19a2e6fc2ddf54f37cf387",
-                "sha256:d2d15a707e7f689531eb4134eccb0f8bf3844bb8255ad50823aa39708d9e6755",
-                "sha256:d4d4519123aac73c93159eb8f61db9682393862dd669e7eae034ecd0a35eadac",
-                "sha256:d93a44d0104d1b9b10de4e7aadf747f6efc1d7ec5ed0aa3f233a720725dd31bd",
-                "sha256:dad638cde3a5357ad3163b80b3127df61fb5b5e34e9e05a87697144400ba03c7",
-                "sha256:e0773cd5c438ffcd5dbff91cdd503574f88a4b960e70cedeb67736583a17a918",
-                "sha256:e172c1ee40507ae3b8d220f4048aaca204f203e1e4197e8e652f5c814f61d1aa",
-                "sha256:e4af8bc5c3fcc2192c266421c6aa2daab1a18e002cb8e66ef672030e46ae25cf",
-                "sha256:e57d94a1552af67f67b27db5dba0b03783ea69d5ca2af2f40e098f0ba3ce3f5f",
-                "sha256:e6b2ba4ff53dd5f57d728095e3def7375eb19c90621ce3b41b256de84ec61cfd",
-                "sha256:e8be562e2ce3e481d9209b6f254c3d7c5ff920eb256aba2380d2fb5ba75d4f87",
-                "sha256:e8ed3bd2b8bb3277710843ca63e4f5c3ee6f8f80b083be5b19a7a9905420d11e",
-                "sha256:e998efb190653f70e0f30d92b39fc645145369a4823bee46af8ddfc244aa969d",
-                "sha256:eaa3ea15025b56076d806b248948612289b093e8dcda8d013776b3848dffff15",
-                "sha256:f4ce006215497993ae77c612c1883ca4f3973899573ce0c52fee91f0d39c4561",
-                "sha256:f7b79f0f302396d8e0d444826ceb3d07b61977793886ebae04e82796c02e42dc",
-                "sha256:f94163ebe2d5546e6a5977e96d83621f4689c1054053428cf8d4c28b10f92f69",
-                "sha256:f988daf25f31726d5b9d0be6af636ca9000898f9ea43a57eac594daea25b0948",
-                "sha256:fbcd613a5e76b1495fc24db9c37a6b7ee5f214fd85979187ec4e032abfc12ded",
-                "sha256:fe454aeab348c42f56d6f7434ff758a3ef90787ac81b9ad5a363cd61b90a1b0b"
+                "sha256:015a0498bde85364abc53fcc713af962dd4555391929736d9c0ff2c555436a03",
+                "sha256:06a484097af32caf1cfffadd60c3ca140c9e52b40a551fb1f6f0fdfd6f7f8977",
+                "sha256:0a65131f7f731ecf7e3dd27f09d877aff3000a79a446caaa2c0d8d0ec0bc7186",
+                "sha256:0f5d3e4e0937dad05585e9bd772bbdf0ca40cd8b2f54789d7a1f3091b608118c",
+                "sha256:156c52eca2b20f9839723bef0b929a290f6c2f1c98ccb24e82f58f96f3c16007",
+                "sha256:1e25f6c8c46cf1ed8237f610abb231093a748c97d6c2c092789a7cad7e7ef290",
+                "sha256:20e7d0e3488cc0f0dbe360731b7fe32e1f2df46bf2de2db3317d301efb93084c",
+                "sha256:23605d7fc67bc7daa0d263b3a26de3375cfcc0b51ab7de5026625415c05b6fed",
+                "sha256:2c8e078d0b9f85212801c41bd9eec8122003929686b0ee33360ffbfdf1a189ab",
+                "sha256:2d6ce4d3828d79044ed08994e196c20f69c18133ed8a4286afe3e98989adeeac",
+                "sha256:3498e72ff2610b049b97bb81d1ea6e7bfa5b7a45efb3f255d77ec2fa2bc91653",
+                "sha256:367e03f1484ce471c94e731b98f5e4a05b43e7188b16692998e1cc89fd1159a5",
+                "sha256:36a05bf59a515cfb07f3f83373c527fff2ecaa77eaf30c968c788aea582070a1",
+                "sha256:3756b44bf247e422a2e47a38f25d03cf4a5ed539fdc2be3c60043e872e6ff13d",
+                "sha256:39a947ff02d9a85673f5ce1f6f34059e24c714a797440485bd81b2c3cb69a7ff",
+                "sha256:3af8e3bcd630f905efbdfe7a51b51fc1ca3c9dca8b155f841925f3ad41685d41",
+                "sha256:3d113b433bc817adf845689a051363777835577858263ec4325d1934fcb7e394",
+                "sha256:3f0c92e3fa826425c73acafb31e022a719c85423847a9433d3a9e61e4ac97543",
+                "sha256:40fd49ef6964b1c90c0bea63cd184f6d0b36e59144a080e8b3ac2c4c06bf6bf2",
+                "sha256:41c8bfd27191928bae6fd2b66872965532267785094a03c0ee5f358d9dba51c2",
+                "sha256:427fc60a188e345534f35b0aa76f7640c5ddf0354f1c9ad826a2bc086282982d",
+                "sha256:44ff8c673cab50be46784e0aec62aa6f0ca9ea765e2b0690e8945d0cd950dcaf",
+                "sha256:48af026a2b1569666da42a478248a1f03f4e2350a34eb661afe3cb45429ca1d7",
+                "sha256:499f8a12767dd28b98ab6b7c7da7d294564e4c9024a2aaa5d0b0b98a8bef2f92",
+                "sha256:4a4f0645a0ec03b229fb04f2e66bdbcb1ffd341a70d6c86c3ee015ffdcd70fad",
+                "sha256:4d1d24d71b6209bc0124286932c4f0660c1103cb996fe34cb374bc12ac251940",
+                "sha256:4dabda7f42c548f98d8e07e390bda2953fc58302c0e07ded7b3fe0637e7ecd2f",
+                "sha256:546a0bb8e5a657cadf0da290b30ccd561cb89c256a5421ab8d5eb12eaf087349",
+                "sha256:56aacf4bf65f575c0392be958aceff719d850950bb6af7d804b32d4bc293159c",
+                "sha256:5afe44da46b48c75169e622a532dca3fe585343c0577cfd7c18ecd3f1200305d",
+                "sha256:5ec374d0f1e7d43ef48a4ff643600833d7a325ecc6933b4d6ad9282f55751cf7",
+                "sha256:61004587a09b5b385e43d95ffe3a76c9d934dfd79ea38272d5c20ddfba8eab8f",
+                "sha256:672c60cce1f8026ae32c651f877aa64f342876083a36a4b1ff91bc876aaf0e34",
+                "sha256:69550c3c053b8f135ceac97b85dc1b2bc54b7613a966f550f32b43bed81c788a",
+                "sha256:6bb6c83d7a65dd3065566c77425ba72df96982174e8ef613d809052d68ae77ab",
+                "sha256:6c3809740124bbc777d29e3ae53de24f4c13fd5e62878086a8feadf0dcb654a5",
+                "sha256:701b7cefffc25de1b7ddfae6505da70a3b3a11e312c2e2b33b09e180bbceb43d",
+                "sha256:70ef7288d1cb1ad16e02d101ea43bb0e392d985d60b9b0035aee80663530960d",
+                "sha256:71db8498e329cef3588b0617f762a3fe31d899872e76a68ce2840e35a1318a5b",
+                "sha256:73682225faa973ee56743f0fcd36bfcbfec503be258e0e420fb34313f52f1e7b",
+                "sha256:7442500fcce71669953ca959682dcd47452bc3f9c95c8d88315874aeabec9f82",
+                "sha256:7a8bd6612fb35487e9ab329bb37b3df44f58baf752010dde9282593edbfed7e7",
+                "sha256:7cbfde62516435ca198220aff048a8793383cb7047c7b88714a061968bca786d",
+                "sha256:80ead7774392efbd95f9f701155048f9ca26cf55133db6f5bb5a0ec69376bda5",
+                "sha256:81b7ce050f26fc1daaaa0d24e320815306736d14608e1ba31920e693a7ca9afb",
+                "sha256:8af5a687c0fb4357c230eec8a57ca07d3172faa3cb69beb0cbad40672ae6fa4b",
+                "sha256:9084e68bedbd665c7e9241a7b597c28f4775edeb3941bf608ecb38732a5f8fb5",
+                "sha256:928208dfecc563be59ae91868d1658e78809cb1e6a0bd74960a96c915db6390c",
+                "sha256:98779cbe9068dd7734cc3210693894d5cc9b156920e9c336f10fb99f46bebbd8",
+                "sha256:98fe771f36ee9d3a1f5741424a956a2ba9651d9508a9f64a024b57f2cf796414",
+                "sha256:9fc8736fc3e0c5aad435520873b9d2e27ddcc5a830b07e00e9c4d3a61ded9675",
+                "sha256:a4170f3016b621e3200ebfcc18de6f50eb8e8fc1303e16324b1f5625afd51b57",
+                "sha256:a9761e425e79d23797fa0bec2d781dbadb9fe5dcc2bf69030855f5e393c3bec8",
+                "sha256:ae092f0daaeece2acdd6ec46e2ab307d8d6f22b01ecca14dc6078844dbd88339",
+                "sha256:b0004dded9d86f129961326e980420187640fb7ba65a184009429861c1d09df7",
+                "sha256:b031395b4b9c3085d9ea1ce89896ab01a65fc63172b2bfda5dd318fefe5e2f93",
+                "sha256:b1644f8b8e19a242c3047a089541067248a651038cabb9fcab3c13eb1dfcd757",
+                "sha256:b234d08786884f5c8d55dfebb839cfbd846d812e3a052c39ca7e8ce7055fed68",
+                "sha256:b2b9051e40b7b649a9a2a38fb223ca6a593d332012df885746b81968948f9435",
+                "sha256:b85b63757ade2439c8d7d71842c40d42c0ab3b69279ed02afbd3b1635f7d2b4b",
+                "sha256:b8737c9b3fd944d856faafa92c95f6198649ad57987935b6d965d086938be917",
+                "sha256:b8bd31aaad8a80a7302730676cec26bea3ef1fd9835875aa47fea073aca9fe05",
+                "sha256:bd1b4531a66da6dde1974662c1fd6fb1a2f27e40542e3df5e5e5dbab8ea4aee7",
+                "sha256:bfb9c45d502ab38c0f4edf98a678694ae0f345613ef4900ade98c71f64db4d78",
+                "sha256:c3f9e322b1ebeebd44e3d9d2d9b124e0c550c1ef41bd552afdcdd719516ee41a",
+                "sha256:c416409646c793c46370f0f1859253302ee70aeda5278c2a0ca41462f8ec1244",
+                "sha256:c59f233f38b6a49d5e4ddf16be910a5bbf36a2989b6b2c8591853fb9f5a5e691",
+                "sha256:cd96e9cb0e2baa294e6d572207d9731c3bb8e2511f1ff70f2bf17266b4488bd9",
+                "sha256:d28c7ef1deb3c3ac5f5290176ca3d501daa97c2e1f7443bf5d8b61ac651794b2",
+                "sha256:d49efdce2086c2c506af20ed18a1115b40af7aad6d4ee27cb31d7c810585a3f2",
+                "sha256:d72493a14a3e89564b1a6c7400b9b40621e8f4692410706ef27c66aeadc7b431",
+                "sha256:d80acb20c7f268fe3150ac0be6a6b798062af56a1795eef855b26c9eae11a99c",
+                "sha256:d86b79c707fc7025d967af71db652429a06a8179175e45bd2e9f17b8af6f5949",
+                "sha256:da16f9cd62c6fde74683be1b28c28ef865e706da13e3bee4ba836fcc520de0cc",
+                "sha256:deebb296df92e082b6d0171a7d6227b503e2897cea4f8bdd3d708094974d4cf6",
+                "sha256:dfebc90273ae2beb813d8118a2bfffb5a5a81ac054fbfd061ea18fd0a81db0ac",
+                "sha256:e3b9bb5fdbe284c7b61c5d82c76688e52bbaf48ab1e53de98c072cc696fa331f",
+                "sha256:e5bfafda019ecc6202af6f3cf08220fa66af9612ba16ef831033ae3ac7bd1f89",
+                "sha256:e71100818943422d1fbbe460e7be7fc4f2d2ba9371b2a745eb09e29ef0493f4a",
+                "sha256:ed016e278c5c4633270903c7cf3b9dfb0bd293b7335e43fe695cb95541da53c9",
+                "sha256:ed04c47dfaab98fcda0b748af9ee6fe8c888a0a0fbd13720e0f0221671e387e1",
+                "sha256:f3661decef5f9ff7ab50edbef463bf7dc717621b56755dbae5458a946a033b10",
+                "sha256:f399269d20ef1dd910331f9ad49e8510c3ba2aa657b623293b536038f266a5c5",
+                "sha256:fe525be0392d493558a2b10d764bcaae9850cc262b417176a8b001f16e085fc6",
+                "sha256:fe627de8fe8ddfa8b6477bda4ae5d5843ad1a0c83601dcff72247039465cc901"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.2.0"
+            "markers": "python_version >= '3.7'",
+            "version": "==3.3.0"
+        },
+        "yarg": {
+            "hashes": [
+                "sha256:4f9cebdc00fac946c9bf2783d634e538a71c7d280a4d806d45fd4dc0ef441492",
+                "sha256:55695bf4d1e3e7f756496c36a69ba32c40d18f821e38f61d028f6049e5e15911"
+            ],
+            "version": "==0.1.9"
         },
         "yarl": {
             "hashes": [
@@ -2652,11 +2692,11 @@
         },
         "click": {
             "hashes": [
-                "sha256:2739815aaa5d2c986a88f1e9230c55e17f0caad3d958a5e13ad0797c166db9e3",
-                "sha256:b97d0c74955da062a7d4ef92fadb583806a585b2ea81958a81bd72726cbb8e37"
+                "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd",
+                "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.1.4"
+            "version": "==8.1.6"
         },
         "coverage": {
             "hashes": [
@@ -2813,19 +2853,19 @@
         },
         "pathspec": {
             "hashes": [
-                "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687",
-                "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"
+                "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20",
+                "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.11.1"
+            "version": "==0.11.2"
         },
         "platformdirs": {
             "hashes": [
-                "sha256:cec7b889196b9144d088e4c57d9ceef7374f6c39694ad1577a0aab50d27ea28c",
-                "sha256:f87ca4fcff7d2b0f81c6a748a77973d7af0f4d526f98f308477c3c436c74d528"
+                "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d",
+                "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.8.1"
+            "version": "==3.10.0"
         },
         "pluggy": {
             "hashes": [
@@ -2916,10 +2956,10 @@
         },
         "types-urllib3": {
             "hashes": [
-                "sha256:3300538c9dc11dad32eae4827ac313f5d986b8b21494801f1bf97a1ac6c03ae5",
-                "sha256:5dbd1d2bef14efee43f5318b5d36d805a489f6600252bb53626d4bfafd95e27c"
+                "sha256:229b7f577c951b8c1b92c1bc2b2fdb0b49847bd2af6d1cc2a2e3dd340f3bda8f",
+                "sha256:9683bbb7fb72e32bfe9d2be6e04875fbe1b3eeec3cbb4ea231435aa7fd6b4f0e"
             ],
-            "version": "==1.26.25.13"
+            "version": "==1.26.25.14"
         },
         "typing-extensions": {
             "hashes": [

From f5833951ad06a6a79162ba3a50768170c9d5592b Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Fri, 4 Aug 2023 22:56:53 -0700
Subject: [PATCH 08/52] update pipfile

---
 Pipfile      |  1 -
 Pipfile.lock | 31 ++++---------------------------
 2 files changed, 4 insertions(+), 28 deletions(-)

diff --git a/Pipfile b/Pipfile
index e71fe91a4e..33ab0b0eaf 100644
--- a/Pipfile
+++ b/Pipfile
@@ -51,7 +51,6 @@ tiktoken = "0.4.0"
 hf-transfer = "0.1.3"
 peft = "0.4.0"
 deepspeed = ">=0.10.0"
-mpi4py = ">=3.1.4"
 
 [dev-packages]
 pytest = "==7.1.3"
diff --git a/Pipfile.lock b/Pipfile.lock
index 9daae5bc60..2e34a90c03 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "18b059325e2f2d74b0582877d64ab84d3c85613753bb1e4724162c9f3c0e093c"
+            "sha256": "f9bd10c7032264b4a539833e1679be1db7891edca43d2b5be7ba3ab6d2178c93"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -936,29 +936,6 @@
             ],
             "version": "==1.6"
         },
-        "mpi4py": {
-            "hashes": [
-                "sha256:152e68aeae40279ca435d15487303159fe3d229337f204141ea805c67b4fbbc3",
-                "sha256:17858f2ebc623220d0120d1fa8d428d033dde749c4bc35b33d81a66ad7f93480",
-                "sha256:1f5f5560211e5b02c388a4e9521acdc5c6ab076e18a60109dc4bcbdb26f0e506",
-                "sha256:32b46ed9d4691ea790d77dde6c5489abd6a4cf6f8e61c35871924c2170bcb7fb",
-                "sha256:4fac4ea098f877ac2d53e6c08f12e84161b960238725ec65795cb4ca391a0649",
-                "sha256:56c79531fc9af9a099c413f38efb20503a84b8355d72a63f0297399f474d88c9",
-                "sha256:5785eecf65669fa97e258d087a86e5b2c5e4dd2b5140e9aee14917cadfe3c876",
-                "sha256:6721a23a2a8ef5a2841ba64926e8e4be0a559344557eef0b2e81821283d56fd1",
-                "sha256:6a7c9dfd1d051aeb1c5c9cc37d8997701732e2e1780f4e188c29c443f6feb6b8",
-                "sha256:6ad6318092adc2bda1567b651d8977f2e418312557cda900805f929f6ccd93b9",
-                "sha256:878ee8c66f3cb4aced2d9d6bde2bbcb5d17d1f90ad0b7407f57e993f6c849a16",
-                "sha256:9e3075e06cea9556cae6562d9194d2c6278c6c8f66af36d8094fb2f77392a90f",
-                "sha256:9ed1b189f859337acc49c2a12a7e13cb2ef3309ce3486b986b082078c4e7493e",
-                "sha256:a0c7f5cee1ffc19d0d145b24eb06997962fe0a07e3019abf5e1575fb43c4ac03",
-                "sha256:bc102ec71939bff487af247f727216a05c45930d931db3f5b206fad9bfcba9c0",
-                "sha256:d939c895ae97256e80ef6e6d626f51bea187d68ab54dd772971fb134632e7ee6",
-                "sha256:f37859af5a318d1af07d3b51714fd36b7674e7368191578cfd96165136279f2d"
-            ],
-            "index": "pypi",
-            "version": "==3.1.4"
-        },
         "mpmath": {
             "hashes": [
                 "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f",
@@ -1574,11 +1551,11 @@
         },
         "referencing": {
             "hashes": [
-                "sha256:47237742e990457f7512c7d27486394a9aadaf876cbfaa4be65b27b4f4d47c6b",
-                "sha256:c257b08a399b6c2f5a3510a50d28ab5dbc7bbde049bcaf954d43c446f83ab548"
+                "sha256:185d4a29f001c6e8ae4dad3861e61282a81cb01b9f0ef70a15450c45c6513a0d",
+                "sha256:9370c77ceefd39510d70948bbe7375ce2d0125b9c11fd380671d4de959a8e3ce"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.30.0"
+            "version": "==0.30.1"
         },
         "regex": {
             "hashes": [

From cbc50fbeb257e128511dd3154b34abae4995e031 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Tue, 8 Aug 2023 23:59:48 -0700
Subject: [PATCH 09/52] zero init for transformers

---
 llm_studio/src/utils/modeling_utils.py | 38 ++++++++++++--------------
 train.py                               |  7 ++++-
 2 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index a900fd71d4..f4140ab4a1 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -183,21 +183,8 @@ def load_checkpoint(
         logger.info(f"Weights loaded from: {weights_path}")
 
 
-def deepspeed_initialize(
-    model: torch.nn.Module,
-    optimizer: torch.optim.Optimizer,
-    lr_scheduler: torch.optim.lr_scheduler._LRScheduler,
-    training_data: torch.utils.data.Dataset,
-    validating_data: torch.utils.data.Dataset,
-    cfg: Any,
-):
-    mconfig = AutoConfig.from_pretrained(cfg.llm_backbone)
-    if hasattr(mconfig, "hidden_size"):
-        model_hidden_size = mconfig.hidden_size
-    elif hasattr(mconfig, "d_model"):
-        model_hidden_size = mconfig.d_model
-    else:
-        raise Exception(f"deepspeed do not support {cfg.llm_backbone}")
+def get_ds_config(cfg: Any):
+
     ds_config = {
         "fp16": {
             "enabled": True if cfg.architecture.backbone_dtype == "float16" else False,
@@ -211,13 +198,12 @@ def deepspeed_initialize(
         "zero_force_ds_cpu_optimizer": False,
         "zero_optimization": {
             "stage": 3,
-            "overlap_comm": False,
+            "overlap_comm": True,
             "contiguous_gradients": True,
-            "reduce_bucket_size": model_hidden_size * model_hidden_size,
+            "reduce_bucket_size": 1e6,
             # zero3
-            "mics_shard_size": cfg.environment._world_size,
-            "stage3_prefetch_bucket_size": 0.9 * model_hidden_size * model_hidden_size,
-            "stage3_param_persistence_threshold": 10 * model_hidden_size,
+            "stage3_prefetch_bucket_size": 1e6,
+            "stage3_param_persistence_threshold": 1e6,
             "stage3_max_live_parameters": cfg.environment.deepspeed_stage3_max_live_parameters,  # noqa: E501
             "stage3_max_reuse_distance": cfg.environment.deepspeed_stage3_max_reuse_distance,  # noqa: E501
             # zero++
@@ -241,6 +227,18 @@ def deepspeed_initialize(
     #     ds_config["zero_optimization"]["offload_param"] =
     #         {"device": "cpu", "pin_memory": True}
 
+    return ds_config
+
+
+def deepspeed_initialize(
+    model: torch.nn.Module,
+    optimizer: torch.optim.Optimizer,
+    lr_scheduler: torch.optim.lr_scheduler._LRScheduler,
+    training_data: torch.utils.data.Dataset,
+    validating_data: torch.utils.data.Dataset,
+    cfg: Any,
+):
+    ds_config = get_ds_config(cfg)
     model, optimizer, train_dataloader, scheduler = deepspeed.initialize(
         model=model,
         optimizer=optimizer,
diff --git a/train.py b/train.py
index 6fb57d8fd9..33392f2644 100644
--- a/train.py
+++ b/train.py
@@ -10,7 +10,6 @@
 import argparse
 import gc
 import logging
-import shutil
 import sys
 import time
 from distutils import util
@@ -24,6 +23,7 @@
 from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler
 from torch.utils.data import DataLoader
 from tqdm import tqdm
+from transformers.deepspeed import HfDeepSpeedConfig
 
 from llm_studio.src.datasets.text_utils import get_tokenizer
 from llm_studio.src.loggers import MainLogger
@@ -53,6 +53,7 @@
 from llm_studio.src.utils.modeling_utils import (
     check_disk_space,
     deepspeed_initialize,
+    get_ds_config,
     get_number_of_validation_epochs,
     get_optimizer,
     get_scheduler,
@@ -626,6 +627,9 @@ def run(cfg: Any) -> None:
         )
 
     # Prepare model
+    ds_config = get_ds_config(cfg)
+    # keep this object alive.
+    dschf = HfDeepSpeedConfig(ds_config)  # noqa: F841
     with torch.device(cfg.environment._device):
         model = cfg.architecture.model_class(cfg)
         check_disk_space(model, cfg.output_directory)
@@ -643,6 +647,7 @@ def run(cfg: Any) -> None:
             load_checkpoint(cfg, model, strict=cfg.training.epochs == -1)
 
     model.to(cfg.environment._device)
+
     if cfg.training.use_rlhf:
         if cfg.training.offload_reward_model:
             reward_model.to("cpu")

From ffee1c028464f5563ca4cec238448f56093d1ef2 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Wed, 9 Aug 2023 00:38:43 -0700
Subject: [PATCH 10/52] add some new config

---
 .../_deepspeed-reduce-bucket-size.mdx         |  1 +
 ...eed-stage3-param-persistence-threshold.mdx |  1 +
 ..._deepspeed-stage3-prefetch-bucket-size.mdx |  1 +
 .../text_causal_language_modeling_config.py   | 22 ++++++++++++++++++-
 4 files changed, 24 insertions(+), 1 deletion(-)
 create mode 100644 documentation/docs/tooltips/experiments/_deepspeed-reduce-bucket-size.mdx
 create mode 100644 documentation/docs/tooltips/experiments/_deepspeed-stage3-param-persistence-threshold.mdx
 create mode 100644 documentation/docs/tooltips/experiments/_deepspeed-stage3-prefetch-bucket-size.mdx

diff --git a/documentation/docs/tooltips/experiments/_deepspeed-reduce-bucket-size.mdx b/documentation/docs/tooltips/experiments/_deepspeed-reduce-bucket-size.mdx
new file mode 100644
index 0000000000..c6155bcb3d
--- /dev/null
+++ b/documentation/docs/tooltips/experiments/_deepspeed-reduce-bucket-size.mdx
@@ -0,0 +1 @@
+Number of elements reduced/allreduced at a time. Limits the memory required for the allgather for large model sizes. Smaller values use less memory, but make training speed slower.
\ No newline at end of file
diff --git a/documentation/docs/tooltips/experiments/_deepspeed-stage3-param-persistence-threshold.mdx b/documentation/docs/tooltips/experiments/_deepspeed-stage3-param-persistence-threshold.mdx
new file mode 100644
index 0000000000..7763fafdb6
--- /dev/null
+++ b/documentation/docs/tooltips/experiments/_deepspeed-stage3-param-persistence-threshold.mdx
@@ -0,0 +1 @@
+Do not partition parameters smaller than this threshold. Smaller values use less memory, but can greatly increase communication and make training speed slower. (especially latency-bound messages).
diff --git a/documentation/docs/tooltips/experiments/_deepspeed-stage3-prefetch-bucket-size.mdx b/documentation/docs/tooltips/experiments/_deepspeed-stage3-prefetch-bucket-size.mdx
new file mode 100644
index 0000000000..29cb1f4f26
--- /dev/null
+++ b/documentation/docs/tooltips/experiments/_deepspeed-stage3-prefetch-bucket-size.mdx
@@ -0,0 +1 @@
+Maximum number of parameter elements to fetch ahead of use. Smaller values use less memory, but make training speed slower.
\ No newline at end of file
diff --git a/llm_studio/python_configs/text_causal_language_modeling_config.py b/llm_studio/python_configs/text_causal_language_modeling_config.py
index b98b36f152..bab0bd06c7 100644
--- a/llm_studio/python_configs/text_causal_language_modeling_config.py
+++ b/llm_studio/python_configs/text_causal_language_modeling_config.py
@@ -392,6 +392,9 @@ class ConfigNLPCausalLMEnvironment(DefaultConfig):
     compile_model: bool = False
     use_fsdp: bool = False
     use_deepspeed: bool = False
+    deepspeed_reduce_bucket_size: int = 1e6
+    deepspeed_stage3_prefetch_bucket_size: int = 1e6
+    deepspeed_stage3_param_persistence_threshold: int = 1e6
     deepspeed_offload_optimizer: bool = False
     deepspeed_stage3_max_live_parameters: int = 1e9
     deepspeed_stage3_max_reuse_distance: int = 1e9
@@ -424,6 +427,15 @@ def __post_init__(self):
 
         self._possible_values["number_of_workers"] = (1, multiprocessing.cpu_count(), 1)
         self._possible_values["seed"] = possible_values.Number(step=1, min=-1)
+        self._possible_values["deepspeed_reduce_bucket_size"] = possible_values.Number(
+            step=1, min=1e7
+        )
+        self._possible_values[
+            "deepspeed_stage3_prefetch_bucket_size"
+        ] = possible_values.Number(step=1, min=1e7)
+        self._possible_values[
+            "deepspeed_stage3_param_persistence_threshold"
+        ] = possible_values.Number(step=1, min=1e7)
         self._possible_values[
             "deepspeed_stage3_max_live_parameters"
         ] = possible_values.Number(step=1, min=1e7)
@@ -432,11 +444,19 @@ def __post_init__(self):
         ] = possible_values.Number(step=1, min=1e7)
         self._nesting.add(
             [
+                "deepspeed_reduce_bucket_size",
+                "deepspeed_stage3_prefetch_bucket_size",
+                "deepspeed_stage3_param_persistence_threshold",
                 "deepspeed_offload_optimizer",
+            ],
+            [Dependency(key="use_deepspeed", value=False, is_set=False)],
+        )
+        self._nesting.add(
+            [
                 "deepspeed_stage3_max_live_parameters",
                 "deepspeed_stage3_max_reuse_distance",
             ],
-            [Dependency(key="use_deepspeed", value=False, is_set=False)],
+            [Dependency(key="deepspeed_offload_optimizer", value=False, is_set=False)],
         )
 
 

From f40ef52cd66119b628a0aec36976588290c44f0f Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Wed, 9 Aug 2023 06:40:38 -0700
Subject: [PATCH 11/52] fix bug

---
 train.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/train.py b/train.py
index 33392f2644..8ddc356b33 100644
--- a/train.py
+++ b/train.py
@@ -627,9 +627,10 @@ def run(cfg: Any) -> None:
         )
 
     # Prepare model
-    ds_config = get_ds_config(cfg)
-    # keep this object alive.
-    dschf = HfDeepSpeedConfig(ds_config)  # noqa: F841
+    if cfg.environment.use_deepspeed:
+        ds_config = get_ds_config(cfg)
+        # keep this object alive.
+        dschf = HfDeepSpeedConfig(ds_config)  # noqa: F841
     with torch.device(cfg.environment._device):
         model = cfg.architecture.model_class(cfg)
         check_disk_space(model, cfg.output_directory)

From 9cd37ab5889cdb4ff04f797d8c51acf0e4f62a4a Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Thu, 10 Aug 2023 03:46:17 -0700
Subject: [PATCH 12/52] min 1e6

---
 .../text_causal_language_modeling_config.py            | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/llm_studio/python_configs/text_causal_language_modeling_config.py b/llm_studio/python_configs/text_causal_language_modeling_config.py
index bab0bd06c7..8c86e9b0fb 100644
--- a/llm_studio/python_configs/text_causal_language_modeling_config.py
+++ b/llm_studio/python_configs/text_causal_language_modeling_config.py
@@ -428,20 +428,20 @@ def __post_init__(self):
         self._possible_values["number_of_workers"] = (1, multiprocessing.cpu_count(), 1)
         self._possible_values["seed"] = possible_values.Number(step=1, min=-1)
         self._possible_values["deepspeed_reduce_bucket_size"] = possible_values.Number(
-            step=1, min=1e7
+            step=1, min=1e6
         )
         self._possible_values[
             "deepspeed_stage3_prefetch_bucket_size"
-        ] = possible_values.Number(step=1, min=1e7)
+        ] = possible_values.Number(step=1, min=1e6)
         self._possible_values[
             "deepspeed_stage3_param_persistence_threshold"
-        ] = possible_values.Number(step=1, min=1e7)
+        ] = possible_values.Number(step=1, min=1e6)
         self._possible_values[
             "deepspeed_stage3_max_live_parameters"
-        ] = possible_values.Number(step=1, min=1e7)
+        ] = possible_values.Number(step=1, min=1e6)
         self._possible_values[
             "deepspeed_stage3_max_reuse_distance"
-        ] = possible_values.Number(step=1, min=1e7)
+        ] = possible_values.Number(step=1, min=1e6)
         self._nesting.add(
             [
                 "deepspeed_reduce_bucket_size",

From 69e9eb183a55be3a5359d3c954143c2c5908554a Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Thu, 17 Aug 2023 06:20:45 -0700
Subject: [PATCH 13/52] update deepspeed config

---
 llm_studio/src/utils/modeling_utils.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index f4140ab4a1..6a55e3513c 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -200,10 +200,10 @@ def get_ds_config(cfg: Any):
             "stage": 3,
             "overlap_comm": True,
             "contiguous_gradients": True,
-            "reduce_bucket_size": 1e6,
+            "reduce_bucket_size": cfg.environment.deepspeed_reduce_bucket_size,
             # zero3
-            "stage3_prefetch_bucket_size": 1e6,
-            "stage3_param_persistence_threshold": 1e6,
+            "stage3_prefetch_bucket_size": cfg.environment.deepspeed_stage3_prefetch_bucket_size,  # noqa: E501
+            "stage3_param_persistence_threshold": cfg.environment.deepspeed_stage3_param_persistence_threshold,  # noqa: E501
             "stage3_max_live_parameters": cfg.environment.deepspeed_stage3_max_live_parameters,  # noqa: E501
             "stage3_max_reuse_distance": cfg.environment.deepspeed_stage3_max_reuse_distance,  # noqa: E501
             # zero++
@@ -213,8 +213,8 @@ def get_ds_config(cfg: Any):
             # "zero_quantized_gradients": True,
         },
         "steps_per_print": 2000,
-        "train_batch_size": cfg.training.batch_size * cfg.environment._world_size,
-        "train_micro_batch_size_per_gpu": 1,
+        "train_micro_batch_size_per_gpu": cfg.training.batch_size,
+        "gradient_accumulation_steps": cfg.training.grad_accumulation,
         "wall_clock_breakdown": False,
     }
     if cfg.environment.deepspeed_offload_optimizer:

From b0df01608d4364ad83ab2e945fefa2326cd65a04 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@users.noreply.github.com>
Date: Thu, 17 Aug 2023 16:16:56 +0000
Subject: [PATCH 14/52] Update requirements.txt

---
 requirements.txt | 47 ++++++++++++++++++++++++++---------------------
 1 file changed, 26 insertions(+), 21 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index bedbff63d0..c9f2425e64 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,43 +5,45 @@ aiohttp==3.8.5 ; python_version >= '3.6'
 aiosignal==1.3.1 ; python_version >= '3.7'
 anyio==3.7.1 ; python_version >= '3.7'
 arrow==1.2.3 ; python_version >= '3.6'
-async-timeout==4.0.2 ; python_version >= '3.6'
+async-timeout==4.0.3 ; python_version >= '3.7'
 attrs==23.1.0 ; python_version >= '3.7'
 beautifulsoup4==4.12.2
 bitsandbytes==0.41.1
 bleach==6.0.0 ; python_version >= '3.7'
 blessed==1.20.0 ; python_version >= '2.7'
 bokeh==3.2.1
-boto3==1.28.19
-botocore==1.31.19 ; python_version >= '3.7'
+boto3==1.28.28
+botocore==1.31.28 ; python_version >= '3.7'
 bravado==11.0.3 ; python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_full_version != '3.5.0'
 bravado-core==6.1.0 ; python_version >= '3.7'
 certifi==2023.7.22 ; python_version >= '3.6'
 charset-normalizer==3.2.0 ; python_full_version >= '3.7.0'
 click==8.1.6 ; python_version >= '3.7'
-cmake==3.27.0
+cmake==3.27.2
 colorama==0.4.6 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
 contourpy==1.1.0 ; python_version >= '3.8'
 coolname==2.2.0
 cramjam==2.7.0 ; python_version >= '3.7'
-datasets==2.14.3
+datasets==2.14.4
+deepspeed==0.10.0
 dill==0.3.7
 docopt==0.6.2
 einops==0.6.1
-exceptiongroup==1.1.2 ; python_version < '3.11'
+exceptiongroup==1.1.3 ; python_version < '3.11'
 fastparquet==2023.7.0
 filelock==3.12.2 ; python_version >= '3.7'
 fqdn==1.5.1
 frozenlist==1.4.0 ; python_version >= '3.8'
 fsspec==2023.6.0 ; python_version >= '3.8'
-future==0.18.3 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+future==0.18.3 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
 gitdb==4.0.10 ; python_version >= '3.7'
 gitpython==3.1.32 ; python_version >= '3.7'
 gputil==1.4.0
-greenlet==2.0.2 ; python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))
+greenlet==2.0.2 ; python_version >= '3' and (platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32'))))))
 h11==0.14.0 ; python_version >= '3.7'
 h2o-wave==0.26.2
 hf-transfer==0.1.3
+hjson==3.1.0
 httpcore==0.17.3 ; python_version >= '3.7'
 httpx==0.24.1 ; python_version >= '3.7'
 huggingface-hub==0.16.4
@@ -50,10 +52,10 @@ inquirer==3.1.3 ; python_version >= '3.8'
 isoduration==20.11.0
 jinja2==3.1.2
 jmespath==1.0.1 ; python_version >= '3.7'
-joblib==1.3.1 ; python_version >= '3.7'
+joblib==1.3.2 ; python_version >= '3.7'
 jsonpointer==2.4
 jsonref==1.1.0 ; python_version >= '3.7'
-jsonschema==4.18.6 ; python_version >= '3.8'
+jsonschema==4.19.0 ; python_version >= '3.8'
 jsonschema-specifications==2023.7.1 ; python_version >= '3.8'
 kaggle==1.5.16
 lit==16.0.6
@@ -65,6 +67,7 @@ multidict==6.0.4 ; python_version >= '3.7'
 multiprocess==0.70.15 ; python_version >= '3.7'
 neptune==1.3.1
 networkx==3.1 ; python_version >= '3.8'
+ninja==1.11.1
 numpy==1.25.2
 oauthlib==3.2.2 ; python_version >= '3.6'
 openai==0.27.8
@@ -76,30 +79,32 @@ pipreqs==0.4.13 ; python_version >= '3.7'
 portalocker==2.7.0 ; python_version >= '3.5'
 protobuf==3.20.3
 psutil==5.9.5 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+py-cpuinfo==9.0.0
 pyarrow==8.0.0
+pydantic==1.10.12 ; python_version >= '3.7'
 pyjwt==2.8.0 ; python_version >= '3.7'
-python-dateutil==2.8.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+python-dateutil==2.8.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
 python-editor==1.0.4
 python-slugify==8.0.1 ; python_version >= '3.7'
 pytz==2023.3
 pyyaml==6.0.1
 readchar==4.0.5 ; python_version >= '3.7'
-referencing==0.30.0 ; python_version >= '3.8'
-regex==2023.6.3 ; python_version >= '3.6'
+referencing==0.30.2 ; python_version >= '3.8'
+regex==2023.8.8 ; python_version >= '3.6'
 requests==2.31.0 ; python_version >= '3.7'
 requests-oauthlib==1.3.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 rfc3339-validator==0.1.4
 rfc3987==1.3.8
 rpds-py==0.9.2 ; python_version >= '3.8'
-s3transfer==0.6.1 ; python_version >= '3.7'
+s3transfer==0.6.2 ; python_version >= '3.7'
 sacrebleu==2.0.0
-safetensors==0.3.1
+safetensors==0.3.2
 scikit-learn==1.3.0
 scipy==1.11.1 ; python_version < '3.13' and python_version >= '3.9'
 sentencepiece==0.1.99
-setuptools==68.0.0 ; python_version >= '3.7'
-simplejson==3.19.1 ; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'
-six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+setuptools==68.1.0 ; python_version >= '3.8'
+simplejson==3.19.1 ; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2'
+six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
 smmap==5.0.0 ; python_version >= '3.6'
 sniffio==1.3.0 ; python_version >= '3.7'
 soupsieve==2.4.1 ; python_version >= '3.7'
@@ -109,7 +114,7 @@ starlette==0.27.0
 swagger-spec-validator==3.0.3 ; python_version >= '3.7'
 sympy==1.12 ; python_version >= '3.8'
 tabulate==0.9.0 ; python_version >= '3.7'
-tenacity==8.2.2
+tenacity==8.2.3
 text-unidecode==1.3
 threadpoolctl==3.2.0 ; python_version >= '3.8'
 tiktoken==0.4.0
@@ -117,8 +122,8 @@ tokenizers==0.13.3
 toml==0.10.2
 torch==2.0.1+cu118
 torchvision==0.15.2+cu118
-tornado==6.3.2 ; python_version >= '3.8'
-tqdm==4.65.0
+tornado==6.3.3 ; python_version >= '3.8'
+tqdm==4.66.1
 transformers==4.31.0
 triton==2.0.0 ; platform_system == 'Linux' and platform_machine == 'x86_64'
 typing-extensions==4.7.1 ; python_version >= '3.7'

From d30b51c36d7dc3de5814bc561db826c196ce615b Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Thu, 17 Aug 2023 20:39:51 -0700
Subject: [PATCH 15/52] remove duplicate code

---
 train.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/train.py b/train.py
index 067ccc88a3..31a3a69a7b 100644
--- a/train.py
+++ b/train.py
@@ -713,10 +713,6 @@ def run(cfg: Any) -> None:
         cfg: config object with all the hyperparameters
     """
 
-    if cfg.environment.use_deepspeed:
-        # To avoid warnings about parallelism in tokenizers
-        os.environ["TOKENIZERS_PARALLELISM"] = "false"
-
     os.makedirs(cfg.output_directory, exist_ok=True)
 
     # Force evaluation if user trains 0 epochs

From 67629eee9dd873992b035626d0fe2ee9f23f7080 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Thu, 17 Aug 2023 20:46:47 -0700
Subject: [PATCH 16/52] throw warning when compile w/ deepspeed

---
 train.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/train.py b/train.py
index 31a3a69a7b..59f7f69302 100644
--- a/train.py
+++ b/train.py
@@ -841,12 +841,18 @@ def run(cfg: Any) -> None:
     if cfg.environment._distributed and not cfg.environment.use_deepspeed:
         model = wrap_model_distributed(model, cfg, cfg.environment.use_fsdp)
 
-    # deepspeed do not support torch.compile
-    if cfg.environment.compile_model and not cfg.environment.use_deepspeed:
-        if cfg.environment._distributed:
-            model.module.backbone = torch.compile(model.module.backbone)
+    if cfg.environment.compile_model
+        # deepspeed do not support torch.compile
+        if cfg.environment.use_deepspeed:
+            logger.warning(
+                "Deepspeed is active, but it doesn't support torch.compile."
+                "Skipping compilation for this experiment."
+            )
         else:
-            model.backbone = torch.compile(model.backbone)
+            if cfg.environment._distributed:
+                model.module.backbone = torch.compile(model.module.backbone)
+            else:
+                model.backbone = torch.compile(model.backbone)
 
     # Force settings when saving best checkpoint
     if cfg.training.save_best_checkpoint:

From 48d7f71359420f426a355f97c1902b27d3c85157 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Thu, 17 Aug 2023 21:35:40 -0700
Subject: [PATCH 17/52] black

---
 llm_studio/src/utils/modeling_utils.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index 93e1d72aeb..0e56f4e5a5 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -199,7 +199,6 @@ def load_checkpoint(
 
 
 def get_ds_config(cfg: Any):
-
     ds_config = {
         "fp16": {
             "enabled": True if cfg.architecture.backbone_dtype == "float16" else False,

From d1efef56fd2a7de4de01dff4a1b511696294803c Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Thu, 17 Aug 2023 23:13:54 -0700
Subject: [PATCH 18/52] integrate deepspeed into wrap_model_distributed

---
 train.py | 69 ++++++++++++++++++++++++++++++++------------------------
 1 file changed, 40 insertions(+), 29 deletions(-)

diff --git a/train.py b/train.py
index 59f7f69302..3282cd4f51 100644
--- a/train.py
+++ b/train.py
@@ -155,6 +155,9 @@ def run_eval(
 def run_train(
     cfg: Any,
     model: torch.nn.Module,
+    optimizer,
+    scheduler,
+    epoch_steps,
     train_dataloader,
     val_dataloader,
     val_df: pd.DataFrame,
@@ -176,27 +179,6 @@ def run_train(
         Last train batch
     """
 
-    epoch_steps = len(train_dataloader)
-
-    optimizer = get_optimizer(model=model, cfg=cfg)
-    scheduler = get_scheduler(cfg=cfg, optimizer=optimizer, epoch_steps=epoch_steps)
-
-    if cfg.environment.use_deepspeed:
-        (
-            model,
-            optimizer,
-            train_dataloader,
-            val_dataloader,
-            scheduler,
-        ) = deepspeed_initialize(
-            model=model,
-            optimizer=optimizer,
-            lr_scheduler=scheduler,
-            training_data=train_dataloader.dataset,
-            validating_data=val_dataloader.dataset,
-            cfg=cfg,
-        )
-
     scaler: GradScaler | ShardedGradScaler | None = None
     if cfg.environment.mixed_precision:
         if cfg.environment.use_fsdp:
@@ -418,6 +400,9 @@ def run_train(
 def run_train_rlhf(
     cfg: Any,
     model: torch.nn.Module,
+    optimizer,
+    scheduler,
+    epoch_steps,
     train_dataloader,
     val_dataloader,
     val_df: pd.DataFrame,
@@ -439,10 +424,8 @@ def run_train_rlhf(
         Last train batch
     """
 
-    epoch_steps = len(train_dataloader)
-
-    optimizer = get_optimizer(model=model, cfg=cfg)
-    scheduler = get_scheduler(cfg=cfg, optimizer=optimizer, epoch_steps=epoch_steps)
+#     optimizer = get_optimizer(model=model, cfg=cfg)
+#     scheduler = get_scheduler(cfg=cfg, optimizer=optimizer, epoch_steps=epoch_steps)
 
     scaler: GradScaler | ShardedGradScaler | None = None
     if cfg.environment.mixed_precision:
@@ -727,6 +710,14 @@ def run(cfg: Any) -> None:
     else:
         cfg.environment._seed = cfg.environment.seed
 
+    if cfg.environment.use_deepspeed and cfg.environment.use_fsdp:
+        raise ValueError("Deepspeed and FSDP cannot be used at the same time.")
+    if cfg.architecture.backbone_dtype in ["int8", "int4"] and cfg.environment.use_deepspeed:
+        raise ValueError(
+            f"Deepspeed do not support backbone type {cfg.architecture.backbone_dtype}. " +
+            "Please set backbone type to float16 or bfloat16 for using deepspeed."
+        )
+
     # Prepare environment
     if "WORLD_SIZE" in os.environ:
         cfg.environment._distributed = int(os.environ["WORLD_SIZE"]) > 1
@@ -816,7 +807,7 @@ def run(cfg: Any) -> None:
             * cfg.environment._world_size
         )
 
-    # Prepare model
+    # Prepare model and optimizer
     if cfg.environment.use_deepspeed:
         ds_config = get_ds_config(cfg)
         # keep this object alive.
@@ -831,6 +822,10 @@ def run(cfg: Any) -> None:
             load_checkpoint(cfg, model, strict=cfg.training.epochs == -1)
     model.to(cfg.environment._device)
 
+    epoch_steps = len(train_dataloader)
+    optimizer = get_optimizer(model=model, cfg=cfg)
+    scheduler = get_scheduler(cfg=cfg, optimizer=optimizer, epoch_steps=epoch_steps)
+
     if getattr(cfg.architecture, "force_embedding_gradients"):
         for module in model.modules():
             if isinstance(module, torch.nn.Embedding):
@@ -838,10 +833,23 @@ def run(cfg: Any) -> None:
                     param.requires_grad = True
                     param.data = param.data.float()
 
-    if cfg.environment._distributed and not cfg.environment.use_deepspeed:
-        model = wrap_model_distributed(model, cfg, cfg.environment.use_fsdp)
+    if cfg.environment._distributed:
+        (
+            model,
+            optimizer,
+            train_dataloader,
+            val_dataloader,
+            scheduler,
+        ) = wrap_model_distributed(
+            model=model,
+            optimizer=optimizer,
+            lr_scheduler=scheduler,
+            training_data=train_dataloader.dataset,
+            validating_data=val_dataloader.dataset,
+            cfg=cfg,
+        )
 
-    if cfg.environment.compile_model
+    if cfg.environment.compile_model:
         # deepspeed do not support torch.compile
         if cfg.environment.use_deepspeed:
             logger.warning(
@@ -896,6 +904,9 @@ def run(cfg: Any) -> None:
     val_loss, val_metric = train_function(
         cfg=cfg,
         model=model,
+        optimizer=optimizer,
+        scheduler=scheduler,
+        epoch_steps=epoch_steps,
         train_dataloader=train_dataloader,
         val_dataloader=val_dataloader,
         val_df=val_df,

From d6b074814198a047e6e3c036c048bcd6da28e5e3 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Thu, 17 Aug 2023 23:17:17 -0700
Subject: [PATCH 19/52] remove unuse code

---
 train.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/train.py b/train.py
index 3282cd4f51..f9ba299f0e 100644
--- a/train.py
+++ b/train.py
@@ -424,9 +424,6 @@ def run_train_rlhf(
         Last train batch
     """
 
-#     optimizer = get_optimizer(model=model, cfg=cfg)
-#     scheduler = get_scheduler(cfg=cfg, optimizer=optimizer, epoch_steps=epoch_steps)
-
     scaler: GradScaler | ShardedGradScaler | None = None
     if cfg.environment.mixed_precision:
         if cfg.environment.use_fsdp:

From 3f8935954ab8177803449bc5d24e8dc39966ec50 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Thu, 17 Aug 2023 23:44:32 -0700
Subject: [PATCH 20/52] style

---
 llm_studio/src/utils/modeling_utils.py | 37 ++++++++++++--------------
 train.py                               | 10 ++++---
 2 files changed, 23 insertions(+), 24 deletions(-)

diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index 0e56f4e5a5..4a82208ae6 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -244,7 +244,7 @@ def get_ds_config(cfg: Any):
     return ds_config
 
 
-def deepspeed_initialize(
+def wrap_model_distributed(
     model: torch.nn.Module,
     optimizer: torch.optim.Optimizer,
     lr_scheduler: torch.optim.lr_scheduler._LRScheduler,
@@ -252,24 +252,7 @@ def deepspeed_initialize(
     validating_data: torch.utils.data.Dataset,
     cfg: Any,
 ):
-    ds_config = get_ds_config(cfg)
-    model, optimizer, train_dataloader, scheduler = deepspeed.initialize(
-        model=model,
-        optimizer=optimizer,
-        lr_scheduler=lr_scheduler,
-        training_data=training_data,
-        config_params=ds_config,
-    )
-    _, _, val_dataloader, _ = deepspeed.initialize(
-        model=torch.nn.Linear(1, 1),
-        training_data=validating_data,
-        config_params=ds_config,
-    )
-    return model, optimizer, train_dataloader, val_dataloader, scheduler
-
-
-def wrap_model_distributed(model: torch.nn.Module, cfg: Any, fsdp: bool):
-    if fsdp:
+    if cfg.environment.use_fsdp:
         auto_wrap_policy = None
 
         mixed_precision_policy = None
@@ -290,6 +273,20 @@ def wrap_model_distributed(model: torch.nn.Module, cfg: Any, fsdp: bool):
             # use_orig_params=False
             limit_all_gathers=True,
         )
+    elif cfg.environment.use_deepspeed:
+        ds_config = get_ds_config(cfg)
+        model, optimizer, train_dataloader, scheduler = deepspeed.initialize(
+            model=model,
+            optimizer=optimizer,
+            lr_scheduler=lr_scheduler,
+            training_data=training_data,
+            config_params=ds_config,
+        )
+        _, _, val_dataloader, _ = deepspeed.initialize(
+            model=torch.nn.Linear(1, 1),
+            training_data=validating_data,
+            config_params=ds_config,
+        )
     else:
         find_unused_parameters = cfg.environment.find_unused_parameters
         if getattr(cfg.architecture, "gradient_checkpointing", None):
@@ -300,7 +297,7 @@ def wrap_model_distributed(model: torch.nn.Module, cfg: Any, fsdp: bool):
             find_unused_parameters=find_unused_parameters,
         )
 
-    return model
+    return model, optimizer, train_dataloader, val_dataloader, scheduler
 
 
 def get_optimizer(model: torch.nn.Module, cfg: Any) -> torch.optim.Optimizer:
diff --git a/train.py b/train.py
index f9ba299f0e..3c9eafa468 100644
--- a/train.py
+++ b/train.py
@@ -52,7 +52,6 @@
 )
 from llm_studio.src.utils.modeling_utils import (
     check_disk_space,
-    deepspeed_initialize,
     get_ds_config,
     get_number_of_validation_epochs,
     get_optimizer,
@@ -709,10 +708,13 @@ def run(cfg: Any) -> None:
 
     if cfg.environment.use_deepspeed and cfg.environment.use_fsdp:
         raise ValueError("Deepspeed and FSDP cannot be used at the same time.")
-    if cfg.architecture.backbone_dtype in ["int8", "int4"] and cfg.environment.use_deepspeed:
+    if (
+        cfg.architecture.backbone_dtype in ["int8", "int4"]
+        and cfg.environment.use_deepspeed
+    ):
         raise ValueError(
-            f"Deepspeed do not support backbone type {cfg.architecture.backbone_dtype}. " +
-            "Please set backbone type to float16 or bfloat16 for using deepspeed."
+            f"Deepspeed do not support backbone type {cfg.architecture.backbone_dtype}."
+            + " Please set backbone type to float16 or bfloat16 for using deepspeed."
         )
 
     # Prepare environment

From 5c253f200ffbdcdf3ab946d514031f165931b5fc Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Fri, 18 Aug 2023 00:33:15 -0700
Subject: [PATCH 21/52] fix bug

---
 llm_studio/src/utils/modeling_utils.py | 12 ++++++------
 train.py                               |  4 ++--
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index 4a82208ae6..96e9e1a18f 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -248,8 +248,8 @@ def wrap_model_distributed(
     model: torch.nn.Module,
     optimizer: torch.optim.Optimizer,
     lr_scheduler: torch.optim.lr_scheduler._LRScheduler,
-    training_data: torch.utils.data.Dataset,
-    validating_data: torch.utils.data.Dataset,
+    train_dataloader: torch.utils.data.DataLoader,
+    val_dataloader: torch.utils.data.DataLoader,
     cfg: Any,
 ):
     if cfg.environment.use_fsdp:
@@ -275,16 +275,16 @@ def wrap_model_distributed(
         )
     elif cfg.environment.use_deepspeed:
         ds_config = get_ds_config(cfg)
-        model, optimizer, train_dataloader, scheduler = deepspeed.initialize(
+        model, optimizer, train_dataloader, lr_scheduler = deepspeed.initialize(
             model=model,
             optimizer=optimizer,
             lr_scheduler=lr_scheduler,
-            training_data=training_data,
+            training_data=train_dataloader.dataset,
             config_params=ds_config,
         )
         _, _, val_dataloader, _ = deepspeed.initialize(
             model=torch.nn.Linear(1, 1),
-            training_data=validating_data,
+            training_data=val_dataloader.dataset,
             config_params=ds_config,
         )
     else:
@@ -297,7 +297,7 @@ def wrap_model_distributed(
             find_unused_parameters=find_unused_parameters,
         )
 
-    return model, optimizer, train_dataloader, val_dataloader, scheduler
+    return model, optimizer, train_dataloader, val_dataloader, lr_scheduler
 
 
 def get_optimizer(model: torch.nn.Module, cfg: Any) -> torch.optim.Optimizer:
diff --git a/train.py b/train.py
index 3c9eafa468..ba248ce754 100644
--- a/train.py
+++ b/train.py
@@ -843,8 +843,8 @@ def run(cfg: Any) -> None:
             model=model,
             optimizer=optimizer,
             lr_scheduler=scheduler,
-            training_data=train_dataloader.dataset,
-            validating_data=val_dataloader.dataset,
+            train_dataloader=train_dataloader,
+            val_dataloader=val_dataloader,
             cfg=cfg,
         )
 

From 9ff717f9e1943db2ede48ff883850442413743a5 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Fri, 18 Aug 2023 01:04:00 -0700
Subject: [PATCH 22/52] fix bug

---
 train.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/train.py b/train.py
index ba248ce754..93dd860aa3 100644
--- a/train.py
+++ b/train.py
@@ -356,25 +356,26 @@ def run_train(
                 val_loss, val_metric = run_eval(
                     cfg=cfg, model=model, val_dataloader=val_dataloader, val_df=val_df
                 )
-                if cfg.environment._local_rank == 0:
-                    if cfg.training.save_best_checkpoint:
-                        if objective_op(val_metric, best_val_metric):
-                            checkpoint_path = cfg.output_directory
+                if cfg.training.save_best_checkpoint:
+                    if objective_op(val_metric, best_val_metric):
+                        checkpoint_path = cfg.output_directory
+                        if cfg.environment._local_rank == 0:
                             logger.info(
                                 f"Saving best model checkpoint: "
                                 f"val_{cfg.prediction.metric} {best_val_metric:.5} -> "
                                 f"{val_metric:.5} to {checkpoint_path}"
                             )
-                            save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
-                            best_val_metric = val_metric
-                    else:
-                        checkpoint_path = cfg.output_directory
+                        save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
+                        best_val_metric = val_metric
+                else:
+                    checkpoint_path = cfg.output_directory
+                    if cfg.environment._local_rank == 0:
                         logger.info(
                             f"Saving last model checkpoint: "
                             f"val_loss {val_loss:.5}, val_{cfg.prediction.metric} "
                             f"{val_metric:.5} to {checkpoint_path}"
                         )
-                        save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
+                    save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
 
                 model.train()
 

From b3495d4f9b1795afff51e14fceb333da8b51d6a0 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Fri, 18 Aug 2023 10:02:52 -0700
Subject: [PATCH 23/52] max token len to 16k

---
 .../python_configs/text_causal_language_modeling_config.py  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llm_studio/python_configs/text_causal_language_modeling_config.py b/llm_studio/python_configs/text_causal_language_modeling_config.py
index 12be6bb0e3..880cc6ccaf 100644
--- a/llm_studio/python_configs/text_causal_language_modeling_config.py
+++ b/llm_studio/python_configs/text_causal_language_modeling_config.py
@@ -227,9 +227,9 @@ class ConfigNLPCausalLMTokenizer(DefaultConfig):
 
     def __post_init__(self):
         super().__post_init__()
-        self._possible_values["max_length_prompt"] = (32, 8192, 32)
-        self._possible_values["max_length_answer"] = (32, 8192, 32)
-        self._possible_values["max_length"] = (32, 8192, 32)
+        self._possible_values["max_length_prompt"] = (32, 1024 * 16, 32)
+        self._possible_values["max_length_answer"] = (32, 1024 * 16, 32)
+        self._possible_values["max_length"] = (32, 1024 * 16, 32)
         self._possible_values["padding_quantile"] = (0, 1, 0.01)
         self._padding_side = "left"
 

From 7b7853868f19d6e483462e20b5c14f0767535415 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Sun, 20 Aug 2023 23:45:06 -0700
Subject: [PATCH 24/52] deepspeed save lora

---
 llm_studio/src/utils/modeling_utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index 96e9e1a18f..d6fd0ab63f 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -70,6 +70,8 @@ def check_disk_space(model: torch.nn.Module, path: str):
             model_size_in_bytes += param.numel() * 4
             logger.warning(f"Unsupported data type: {param.data.dtype}")
 
+    if cfg.environment.use_deepspeed:
+        model_size_in_bytes *= 2  # need double space for converting deepspeed engine.
     if model_size_in_bytes * 1.03 < free:  # leave a 3% margin here.
         logger.info("Enough space available for saving model weights.")
     else:
@@ -95,6 +97,8 @@ def save_checkpoint(model: torch.nn.Module, path: str, cfg: Any):
     if cfg.environment.use_deepspeed:
         if path is not None:
             # gather model params from all ranks
+            if hasattr(cfg.training, "lora") and cfg.training.lora:
+                model.backbone.save_pretrained(path)
             model.save_checkpoint(os.path.join(path, "ds_checkpoint"))
             if cfg.environment._local_rank == 0:
                 # load to cpu

From 892f47cbfe6c56af96ffae737d5234fc825db332 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Sun, 20 Aug 2023 23:45:46 -0700
Subject: [PATCH 25/52] update get optimizer

---
 llm_studio/src/utils/modeling_utils.py | 112 ++++++++++++-------------
 1 file changed, 52 insertions(+), 60 deletions(-)

diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index d6fd0ab63f..fbefb30bcb 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -314,66 +314,58 @@ def get_optimizer(model: torch.nn.Module, cfg: Any) -> torch.optim.Optimizer:
     Returns:
         Optimizer
     """
-    if cfg.environment.use_deepspeed and cfg.training.lora:
-        logger.info(
-            "Deepspeed /w Lora training do not support differential learning rate."
-        )
-        optimizer = Optimizers.get(cfg.training.optimizer)(
-            model.parameters(), lr=cfg.training.learning_rate
-        )
-    else:
-        no_decay = ["bias", "LayerNorm.weight"]
-        differential_layers = cfg.training.differential_learning_rate_layers
-        optimizer = Optimizers.get(cfg.training.optimizer)(
-            [
-                {
-                    "params": [
-                        param
-                        for name, param in model.named_parameters()
-                        if (not any(layer in name for layer in differential_layers))
-                        and (not any(nd in name for nd in no_decay))
-                        # and param.requires_grad
-                    ],
-                    "lr": cfg.training.learning_rate,
-                    "weight_decay": cfg.training.weight_decay,
-                },
-                {
-                    "params": [
-                        param
-                        for name, param in model.named_parameters()
-                        if (not any(layer in name for layer in differential_layers))
-                        and (any(nd in name for nd in no_decay))
-                        # and param.requires_grad
-                    ],
-                    "lr": cfg.training.learning_rate,
-                    "weight_decay": 0,
-                },
-                {
-                    "params": [
-                        param
-                        for name, param in model.named_parameters()
-                        if (any(layer in name for layer in differential_layers))
-                        and (not any(nd in name for nd in no_decay))
-                        # and param.requires_grad
-                    ],
-                    "lr": cfg.training.differential_learning_rate,
-                    "weight_decay": cfg.training.weight_decay,
-                },
-                {
-                    "params": [
-                        param
-                        for name, param in model.named_parameters()
-                        if (any(layer in name for layer in differential_layers))
-                        and (any(nd in name for nd in no_decay))
-                        # and param.requires_grad
-                    ],
-                    "lr": cfg.training.differential_learning_rate,
-                    "weight_decay": 0,
-                },
-            ],
-            lr=cfg.training.learning_rate,
-            weight_decay=cfg.training.weight_decay,
-        )
+    no_decay = ["bias", "LayerNorm.weight"]
+    differential_layers = cfg.training.differential_learning_rate_layers
+    optimizer = Optimizers.get(cfg.training.optimizer)(
+        [
+            {
+                "params": [
+                    param
+                    for name, param in model.named_parameters()
+                    if (not any(layer in name for layer in differential_layers))
+                    and (not any(nd in name for nd in no_decay))
+                    and param.requires_grad
+                ],
+                "lr": cfg.training.learning_rate,
+                "weight_decay": cfg.training.weight_decay,
+            },
+            {
+                "params": [
+                    param
+                    for name, param in model.named_parameters()
+                    if (not any(layer in name for layer in differential_layers))
+                    and (any(nd in name for nd in no_decay))
+                    and param.requires_grad
+                ],
+                "lr": cfg.training.learning_rate,
+                "weight_decay": 0,
+            },
+            {
+                "params": [
+                    param
+                    for name, param in model.named_parameters()
+                    if (any(layer in name for layer in differential_layers))
+                    and (not any(nd in name for nd in no_decay))
+                    and param.requires_grad
+                ],
+                "lr": cfg.training.differential_learning_rate,
+                "weight_decay": cfg.training.weight_decay,
+            },
+            {
+                "params": [
+                    param
+                    for name, param in model.named_parameters()
+                    if (any(layer in name for layer in differential_layers))
+                    and (any(nd in name for nd in no_decay))
+                    and param.requires_grad
+                ],
+                "lr": cfg.training.differential_learning_rate,
+                "weight_decay": 0,
+            },
+        ],
+        lr=cfg.training.learning_rate,
+        weight_decay=cfg.training.weight_decay,
+    )
 
     return optimizer
 

From f2dfb8926ae163937f0441592eb3f099f6e98fb2 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Sun, 20 Aug 2023 23:55:12 -0700
Subject: [PATCH 26/52] fix check disk

---
 app_utils/sections/experiment.py       | 2 +-
 llm_studio/src/utils/modeling_utils.py | 4 ++--
 train.py                               | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/app_utils/sections/experiment.py b/app_utils/sections/experiment.py
index 99beff907e..097fc4a9b8 100644
--- a/app_utils/sections/experiment.py
+++ b/app_utils/sections/experiment.py
@@ -1765,7 +1765,7 @@ async def experiment_push_to_huggingface_dialog(q: Q, error: str = ""):
                 device=q.client["experiment/display/push_to_huggingface/device"],
             )
 
-        check_disk_space(model.backbone, "./")
+        check_disk_space(model.backbone, "./", False)
 
         huggingface_hub.login(
             q.client["experiment/display/push_to_huggingface/api_key"]
diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index fbefb30bcb..91ef77760f 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -54,7 +54,7 @@ def unwrap_model(model: torch.nn.Module):
     return model
 
 
-def check_disk_space(model: torch.nn.Module, path: str):
+def check_disk_space(model: torch.nn.Module, path: str, use_deepspeed: bool):
     total, used, free = shutil.disk_usage(path)
 
     model_size_in_bytes = 0
@@ -70,7 +70,7 @@ def check_disk_space(model: torch.nn.Module, path: str):
             model_size_in_bytes += param.numel() * 4
             logger.warning(f"Unsupported data type: {param.data.dtype}")
 
-    if cfg.environment.use_deepspeed:
+    if use_deepspeed:
         model_size_in_bytes *= 2  # need double space for converting deepspeed engine.
     if model_size_in_bytes * 1.03 < free:  # leave a 3% margin here.
         logger.info("Enough space available for saving model weights.")
diff --git a/train.py b/train.py
index f7b0f46cf0..d78c8a6a46 100644
--- a/train.py
+++ b/train.py
@@ -814,7 +814,7 @@ def run(cfg: Any) -> None:
         dschf = HfDeepSpeedConfig(ds_config)  # noqa: F841
     with torch.device(cfg.environment._device):
         model = cfg.architecture.model_class(cfg)
-        check_disk_space(model, cfg.output_directory)
+        check_disk_space(model, cfg.output_directory, cfg.environment.use_deepspeed)
 
         # load model weights
         if cfg.architecture.pretrained_weights != "":

From d297ec9cced6dc93471bf1c8452cb4aa46b8ac96 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Sun, 27 Aug 2023 22:18:48 -0700
Subject: [PATCH 27/52] comment out offload CPU

---
 .../text_causal_language_modeling_config.py   | 22 +++++++++----------
 llm_studio/src/utils/modeling_utils.py        | 16 ++++++++------
 2 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/llm_studio/python_configs/text_causal_language_modeling_config.py b/llm_studio/python_configs/text_causal_language_modeling_config.py
index 880cc6ccaf..62bd361382 100644
--- a/llm_studio/python_configs/text_causal_language_modeling_config.py
+++ b/llm_studio/python_configs/text_causal_language_modeling_config.py
@@ -347,9 +347,9 @@ class ConfigNLPCausalLMEnvironment(DefaultConfig):
     deepspeed_reduce_bucket_size: int = 1e6
     deepspeed_stage3_prefetch_bucket_size: int = 1e6
     deepspeed_stage3_param_persistence_threshold: int = 1e6
-    deepspeed_offload_optimizer: bool = False
-    deepspeed_stage3_max_live_parameters: int = 1e9
-    deepspeed_stage3_max_reuse_distance: int = 1e9
+#     deepspeed_offload_optimizer: bool = False
+#     deepspeed_stage3_max_live_parameters: int = 1e9
+#     deepspeed_stage3_max_reuse_distance: int = 1e9
 
     find_unused_parameters: bool = False
     trust_remote_code: bool = True
@@ -403,17 +403,17 @@ def __post_init__(self):
                 "deepspeed_reduce_bucket_size",
                 "deepspeed_stage3_prefetch_bucket_size",
                 "deepspeed_stage3_param_persistence_threshold",
-                "deepspeed_offload_optimizer",
+                # "deepspeed_offload_optimizer",
             ],
             [Dependency(key="use_deepspeed", value=False, is_set=False)],
         )
-        self._nesting.add(
-            [
-                "deepspeed_stage3_max_live_parameters",
-                "deepspeed_stage3_max_reuse_distance",
-            ],
-            [Dependency(key="deepspeed_offload_optimizer", value=False, is_set=False)],
-        )
+        # self._nesting.add(
+        #     [
+        #         "deepspeed_stage3_max_live_parameters",
+        #         "deepspeed_stage3_max_reuse_distance",
+        #     ],
+        #     [Dependency(key="deepspeed_offload_optimizer", value=False, is_set=False)],  # noqa: E501
+        # )
 
 
 @dataclass
diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index 91ef77760f..137d88e4b3 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -222,8 +222,9 @@ def get_ds_config(cfg: Any):
             # zero3
             "stage3_prefetch_bucket_size": cfg.environment.deepspeed_stage3_prefetch_bucket_size,  # noqa: E501
             "stage3_param_persistence_threshold": cfg.environment.deepspeed_stage3_param_persistence_threshold,  # noqa: E501
-            "stage3_max_live_parameters": cfg.environment.deepspeed_stage3_max_live_parameters,  # noqa: E501
-            "stage3_max_reuse_distance": cfg.environment.deepspeed_stage3_max_reuse_distance,  # noqa: E501
+            # zero3 offload cpu
+            # "stage3_max_live_parameters": cfg.environment.deepspeed_stage3_max_live_parameters,  # noqa: E501
+            # "stage3_max_reuse_distance": cfg.environment.deepspeed_stage3_max_reuse_distance,  # noqa: E501
             # zero++
             # "reduce_scatter": True,
             # "zero_quantized_weights": True,
@@ -235,11 +236,12 @@ def get_ds_config(cfg: Any):
         "gradient_accumulation_steps": cfg.training.grad_accumulation,
         "wall_clock_breakdown": False,
     }
-    if cfg.environment.deepspeed_offload_optimizer:
-        ds_config["zero_optimization"]["offload_optimizer"] = {
-            "device": "cpu",
-            "pin_memory": True,
-        }
+    # TODO: Do not enable offload cpu for now.
+    # if cfg.environment.deepspeed_offload_optimizer:
+    #     ds_config["zero_optimization"]["offload_optimizer"] = {
+    #         "device": "cpu",
+    #         "pin_memory": True,
+    #     }
     # TODO: RuntimeError: Tensors must be CUDA and dense
     # if cfg.environment.deepspeed_offload_param:
     #     ds_config["zero_optimization"]["offload_param"] =

From e16cab8faa82facf37ef385fe73329630a485610 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Mon, 28 Aug 2023 06:24:33 -0700
Subject: [PATCH 28/52] Pipfile.lock

---
 Pipfile.lock | 439 ++++++++++++++++++++++++++-------------------------
 1 file changed, 227 insertions(+), 212 deletions(-)

diff --git a/Pipfile.lock b/Pipfile.lock
index 5fae3b1e6a..b04c97eb31 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -204,19 +204,19 @@
         },
         "boto3": {
             "hashes": [
-                "sha256:4a435fdbd77628e3d32cfbc8b6225e779d8f789027fadb6a51fe1b456e15ef54",
-                "sha256:57d618f03bd269ebef6287dd4ed86ddaa1d53a4021008ad3267c6097be17e172"
+                "sha256:580b584e36967155abed7cc9b088b3bd784e8242ae4d8841f58cb50ab05520dc",
+                "sha256:d77415f22bbc14f3d72eaed2fc9f96d161f3ba7686922ad26d6bbc9d4985f3df"
             ],
             "index": "pypi",
-            "version": "==1.28.28"
+            "version": "==1.28.35"
         },
         "botocore": {
             "hashes": [
-                "sha256:1fcfbd23c7f1f66f16c5c1a1e8565ee8ff68429cc0ee9d2acfb1b55739584cbd",
-                "sha256:d6310826e37ba0209e904d691638b8e848342ec17f5187568ca02ad092c55c45"
+                "sha256:7e4534325262f43293a9cc9937cb3f1711365244ffde8b925a6ee862bcf30a83",
+                "sha256:943e1465aad66db4933b06809134bd08c5b05e8eb18c19742ffec82f54769457"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.31.28"
+            "version": "==1.31.35"
         },
         "bravado": {
             "hashes": [
@@ -325,33 +325,26 @@
         },
         "click": {
             "hashes": [
-                "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd",
-                "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"
+                "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28",
+                "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.1.6"
+            "version": "==8.1.7"
         },
         "cmake": {
             "hashes": [
-                "sha256:115d30ca0760e3861d9ad6b3288cd11ee72a785b81227da0c1765d3b84e2c009",
-                "sha256:11fe6129d07982721c5965fd804a4056b8c6e9c4f482ac9e0fe41bb3abc1ab5f",
-                "sha256:197a34dc62ee149ced343545fac67e5a30b93fda65250b065726f86ce92bdada",
-                "sha256:294f008734267e0eee1574ad1b911bed137bc907ab19d60a618dab4615aa1fca",
-                "sha256:30620326b51ac2ce0d8f476747af6367a7ea21075c4d065fad9443904b07476a",
-                "sha256:53e12deb893da935e236f93accd47dbe2806620cd7654986234dc4487cc49652",
-                "sha256:611f9722c68c40352d38a6c01960ab038c3d0419e7aee3bf18f95b23031e0dfe",
-                "sha256:7cd6e2d7d5a1125f8c26c4f65214f8c942e3f276f98c16cb62ae382c35609f25",
-                "sha256:7f7438c60ccc01765b67abfb1797787c3b9459d500a804ed70a4cc181bc02204",
-                "sha256:83611ffd155e270a6b13bbf0cfd4e8688ebda634f448aa2e3734006c745bf33f",
-                "sha256:96ac856c4d6b2104408848f0005a8ab2229d4135b171ea9a03e8c33039ede420",
-                "sha256:980ee19f12c808cb8ddb56fdcee832501a9f9631799d8b4fc625c0a0b5fb4c55",
-                "sha256:ab2e40fe09e76a7ef67da2bbbf7a4cd1f52db4f1c7b6ccdda2539f918830343a",
-                "sha256:afb46ad883b174fb64347802ba5878423551dbd5847bb64669c39a5957c06eb7",
-                "sha256:ca7650477dff2a1138776b28b79c0e99127be733d3978922e8f87b56a433eed6",
-                "sha256:efc338c939d6d435890a52458a260bf0942bd8392b648d7532a72c1ec0764e18",
-                "sha256:f0c64e89e2ea59592980c4fe3821d712fee0e74cf87c2aaec5b3ab9aa809a57c"
-            ],
-            "version": "==3.27.2"
+                "sha256:2b0b53ec2e45cfe9982d0adf833b3519efc328c1e3cffae4d237841a1ed6edf4",
+                "sha256:2f217fb281b068696fdcc4b198de62e9ded8bc0a93877684afc59db3507ccb44",
+                "sha256:37e9cad75184fbefe837311528d026901278b606707e9d14b58e3767d49d0aa6",
+                "sha256:4a1d22ee72dcdc32d0f8bbf5691d2e9367585db8bfeafe7cffa2c4274127a801",
+                "sha256:65b79f1e8b6fa254697ee0b411aa4dff0d2309c1405af3448adf06cbd7ef0ac5",
+                "sha256:871c8b5eaac959f079c2389c7a7f198fa5f86a029e8726fcb1f3e13d030c33e9",
+                "sha256:a3bd8b9d0e294bd2b3ce27a850c9d924aee7e4f4c0bb56d66641cc1544314f58",
+                "sha256:a504815bcba0ece9aafb48a6b7770d6479756fda92f8b62f9ab7ff8a403a12d5",
+                "sha256:c36eb106dec60198264b25d4bd23cd9ea30b0af9200a143ec1db887c095306f7",
+                "sha256:ec4a5bc2376dfc57065bfde6806183b331165f33457b7cc0fc0511260dde7c72"
+            ],
+            "version": "==3.27.4.1"
         },
         "colorama": {
             "hashes": [
@@ -489,10 +482,10 @@
         },
         "deepspeed": {
             "hashes": [
-                "sha256:afb06a97fde2a33d0cbd60a8357a70087c037b9f647ca48377728330c35eff3e"
+                "sha256:6db8f6be222dcdcefa3cae1fdf80aa6512d8a6851d53f3c4232083c70aada798"
             ],
             "index": "pypi",
-            "version": "==0.10.0"
+            "version": "==0.10.1"
         },
         "dill": {
             "hashes": [
@@ -1235,11 +1228,11 @@
         },
         "openai": {
             "hashes": [
-                "sha256:2483095c7db1eee274cebac79e315a986c4e55207bb4fa7b82d185b3a2ed9536",
-                "sha256:e0a7c2f7da26bdbe5354b03c6d4b82a2f34bd4458c7a17ae1a7092c3e397e03c"
+                "sha256:6a3cf8e276d1a6262b50562fbc0cba7967cfebb78ed827d375986b48fdad6475",
+                "sha256:b687761c82f5ebb6f61efc791b2083d2d068277b94802d4d1369efe39851813d"
             ],
             "index": "pypi",
-            "version": "==0.27.8"
+            "version": "==0.27.9"
         },
         "packaging": {
             "hashes": [
@@ -1284,11 +1277,11 @@
         },
         "peft": {
             "hashes": [
-                "sha256:2cf992772a6d703814477e0bdcdadd68cb8ea388111ce2d793dd2ff0e438f357",
-                "sha256:e768fa22d6e9f32aa7e891f0d06f355960278ca4dc0cdd96bff71f6f06269207"
+                "sha256:3cc03049ac6130426d82c22ca54cc6de849e7c6727181393e7751f7f512a9ced",
+                "sha256:82b307830f946bcc1e8e19419a55aac50fcf670a55d6be04eb6adbcbc5c21042"
             ],
             "index": "pypi",
-            "version": "==0.4.0"
+            "version": "==0.5.0"
         },
         "pillow": {
             "hashes": [
@@ -1730,106 +1723,106 @@
         },
         "rpds-py": {
             "hashes": [
-                "sha256:0173c0444bec0a3d7d848eaeca2d8bd32a1b43f3d3fde6617aac3731fa4be05f",
-                "sha256:01899794b654e616c8625b194ddd1e5b51ef5b60ed61baa7a2d9c2ad7b2a4238",
-                "sha256:02938432352359805b6da099c9c95c8a0547fe4b274ce8f1a91677401bb9a45f",
-                "sha256:03421628f0dc10a4119d714a17f646e2837126a25ac7a256bdf7c3943400f67f",
-                "sha256:03975db5f103997904c37e804e5f340c8fdabbb5883f26ee50a255d664eed58c",
-                "sha256:0766babfcf941db8607bdaf82569ec38107dbb03c7f0b72604a0b346b6eb3298",
-                "sha256:07e2c54bef6838fa44c48dfbc8234e8e2466d851124b551fc4e07a1cfeb37260",
-                "sha256:0836d71ca19071090d524739420a61580f3f894618d10b666cf3d9a1688355b1",
-                "sha256:095b460e117685867d45548fbd8598a8d9999227e9061ee7f012d9d264e6048d",
-                "sha256:0e7521f5af0233e89939ad626b15278c71b69dc1dfccaa7b97bd4cdf96536bb7",
-                "sha256:0f2996fbac8e0b77fd67102becb9229986396e051f33dbceada3debaacc7033f",
-                "sha256:1054a08e818f8e18910f1bee731583fe8f899b0a0a5044c6e680ceea34f93876",
-                "sha256:13b602dc3e8dff3063734f02dcf05111e887f301fdda74151a93dbbc249930fe",
-                "sha256:141acb9d4ccc04e704e5992d35472f78c35af047fa0cfae2923835d153f091be",
-                "sha256:14c408e9d1a80dcb45c05a5149e5961aadb912fff42ca1dd9b68c0044904eb32",
-                "sha256:159fba751a1e6b1c69244e23ba6c28f879a8758a3e992ed056d86d74a194a0f3",
-                "sha256:190ca6f55042ea4649ed19c9093a9be9d63cd8a97880106747d7147f88a49d18",
-                "sha256:196cb208825a8b9c8fc360dc0f87993b8b260038615230242bf18ec84447c08d",
-                "sha256:1fcdee18fea97238ed17ab6478c66b2095e4ae7177e35fb71fbe561a27adf620",
-                "sha256:207f57c402d1f8712618f737356e4b6f35253b6d20a324d9a47cb9f38ee43a6b",
-                "sha256:24a81c177379300220e907e9b864107614b144f6c2a15ed5c3450e19cf536fae",
-                "sha256:29cd8bfb2d716366a035913ced99188a79b623a3512292963d84d3e06e63b496",
-                "sha256:2d8b3b3a2ce0eaa00c5bbbb60b6713e94e7e0becab7b3db6c5c77f979e8ed1f1",
-                "sha256:35da5cc5cb37c04c4ee03128ad59b8c3941a1e5cd398d78c37f716f32a9b7f67",
-                "sha256:44659b1f326214950a8204a248ca6199535e73a694be8d3e0e869f820767f12f",
-                "sha256:47c5f58a8e0c2c920cc7783113df2fc4ff12bf3a411d985012f145e9242a2764",
-                "sha256:4bd4dc3602370679c2dfb818d9c97b1137d4dd412230cfecd3c66a1bf388a196",
-                "sha256:4ea6b73c22d8182dff91155af018b11aac9ff7eca085750455c5990cb1cfae6e",
-                "sha256:50025635ba8b629a86d9d5474e650da304cb46bbb4d18690532dd79341467846",
-                "sha256:517cbf6e67ae3623c5127206489d69eb2bdb27239a3c3cc559350ef52a3bbf0b",
-                "sha256:5855c85eb8b8a968a74dc7fb014c9166a05e7e7a8377fb91d78512900aadd13d",
-                "sha256:5a46859d7f947061b4010e554ccd1791467d1b1759f2dc2ec9055fa239f1bc26",
-                "sha256:65a0583c43d9f22cb2130c7b110e695fff834fd5e832a776a107197e59a1898e",
-                "sha256:674c704605092e3ebbbd13687b09c9f78c362a4bc710343efe37a91457123044",
-                "sha256:682726178138ea45a0766907957b60f3a1bf3acdf212436be9733f28b6c5af3c",
-                "sha256:686ba516e02db6d6f8c279d1641f7067ebb5dc58b1d0536c4aaebb7bf01cdc5d",
-                "sha256:6a5d3fbd02efd9cf6a8ffc2f17b53a33542f6b154e88dd7b42ef4a4c0700fdad",
-                "sha256:6aa8326a4a608e1c28da191edd7c924dff445251b94653988efb059b16577a4d",
-                "sha256:700375326ed641f3d9d32060a91513ad668bcb7e2cffb18415c399acb25de2ab",
-                "sha256:71f2f7715935a61fa3e4ae91d91b67e571aeb5cb5d10331ab681256bda2ad920",
-                "sha256:745f5a43fdd7d6d25a53ab1a99979e7f8ea419dfefebcab0a5a1e9095490ee5e",
-                "sha256:79f594919d2c1a0cc17d1988a6adaf9a2f000d2e1048f71f298b056b1018e872",
-                "sha256:7d68dc8acded354c972116f59b5eb2e5864432948e098c19fe6994926d8e15c3",
-                "sha256:7f67da97f5b9eac838b6980fc6da268622e91f8960e083a34533ca710bec8611",
-                "sha256:83b32f0940adec65099f3b1c215ef7f1d025d13ff947975a055989cb7fd019a4",
-                "sha256:876bf9ed62323bc7dcfc261dbc5572c996ef26fe6406b0ff985cbcf460fc8a4c",
-                "sha256:890ba852c16ace6ed9f90e8670f2c1c178d96510a21b06d2fa12d8783a905193",
-                "sha256:8b08605d248b974eb02f40bdcd1a35d3924c83a2a5e8f5d0fa5af852c4d960af",
-                "sha256:8b2eb034c94b0b96d5eddb290b7b5198460e2d5d0c421751713953a9c4e47d10",
-                "sha256:8b9ec12ad5f0a4625db34db7e0005be2632c1013b253a4a60e8302ad4d462afd",
-                "sha256:8c8d7594e38cf98d8a7df25b440f684b510cf4627fe038c297a87496d10a174f",
-                "sha256:8d3335c03100a073883857e91db9f2e0ef8a1cf42dc0369cbb9151c149dbbc1b",
-                "sha256:8d70e8f14900f2657c249ea4def963bed86a29b81f81f5b76b5a9215680de945",
-                "sha256:9039a11bca3c41be5a58282ed81ae422fa680409022b996032a43badef2a3752",
-                "sha256:91378d9f4151adc223d584489591dbb79f78814c0734a7c3bfa9c9e09978121c",
-                "sha256:9251eb8aa82e6cf88510530b29eef4fac825a2b709baf5b94a6094894f252387",
-                "sha256:933a7d5cd4b84f959aedeb84f2030f0a01d63ae6cf256629af3081cf3e3426e8",
-                "sha256:978fa96dbb005d599ec4fd9ed301b1cc45f1a8f7982d4793faf20b404b56677d",
-                "sha256:987b06d1cdb28f88a42e4fb8a87f094e43f3c435ed8e486533aea0bf2e53d931",
-                "sha256:99b1c16f732b3a9971406fbfe18468592c5a3529585a45a35adbc1389a529a03",
-                "sha256:99e7c4bb27ff1aab90dcc3e9d37ee5af0231ed98d99cb6f5250de28889a3d502",
-                "sha256:9c439fd54b2b9053717cca3de9583be6584b384d88d045f97d409f0ca867d80f",
-                "sha256:9ea4d00850ef1e917815e59b078ecb338f6a8efda23369677c54a5825dbebb55",
-                "sha256:9f30d205755566a25f2ae0382944fcae2f350500ae4df4e795efa9e850821d82",
-                "sha256:a06418fe1155e72e16dddc68bb3780ae44cebb2912fbd8bb6ff9161de56e1798",
-                "sha256:a0805911caedfe2736935250be5008b261f10a729a303f676d3d5fea6900c96a",
-                "sha256:a1f044792e1adcea82468a72310c66a7f08728d72a244730d14880cd1dabe36b",
-                "sha256:a216b26e5af0a8e265d4efd65d3bcec5fba6b26909014effe20cd302fd1138fa",
-                "sha256:a987578ac5214f18b99d1f2a3851cba5b09f4a689818a106c23dbad0dfeb760f",
-                "sha256:aad51239bee6bff6823bbbdc8ad85136c6125542bbc609e035ab98ca1e32a192",
-                "sha256:ab2299e3f92aa5417d5e16bb45bb4586171c1327568f638e8453c9f8d9e0f020",
-                "sha256:ab6919a09c055c9b092798ce18c6c4adf49d24d4d9e43a92b257e3f2548231e7",
-                "sha256:b0c43f8ae8f6be1d605b0465671124aa8d6a0e40f1fb81dcea28b7e3d87ca1e1",
-                "sha256:b1440c291db3f98a914e1afd9d6541e8fc60b4c3aab1a9008d03da4651e67386",
-                "sha256:b52e7c5ae35b00566d244ffefba0f46bb6bec749a50412acf42b1c3f402e2c90",
-                "sha256:bf4151acb541b6e895354f6ff9ac06995ad9e4175cbc6d30aaed08856558201f",
-                "sha256:c27ee01a6c3223025f4badd533bea5e87c988cb0ba2811b690395dfe16088cfe",
-                "sha256:c545d9d14d47be716495076b659db179206e3fd997769bc01e2d550eeb685596",
-                "sha256:c5934e2833afeaf36bd1eadb57256239785f5af0220ed8d21c2896ec4d3a765f",
-                "sha256:c7671d45530fcb6d5e22fd40c97e1e1e01965fc298cbda523bb640f3d923b387",
-                "sha256:c861a7e4aef15ff91233751619ce3a3d2b9e5877e0fcd76f9ea4f6847183aa16",
-                "sha256:d25b1c1096ef0447355f7293fbe9ad740f7c47ae032c2884113f8e87660d8f6e",
-                "sha256:d55777a80f78dd09410bd84ff8c95ee05519f41113b2df90a69622f5540c4f8b",
-                "sha256:d576c3ef8c7b2d560e301eb33891d1944d965a4d7a2eacb6332eee8a71827db6",
-                "sha256:dd9da77c6ec1f258387957b754f0df60766ac23ed698b61941ba9acccd3284d1",
-                "sha256:de0b6eceb46141984671802d412568d22c6bacc9b230174f9e55fc72ef4f57de",
-                "sha256:e07e5dbf8a83c66783a9fe2d4566968ea8c161199680e8ad38d53e075df5f0d0",
-                "sha256:e564d2238512c5ef5e9d79338ab77f1cbbda6c2d541ad41b2af445fb200385e3",
-                "sha256:ed89861ee8c8c47d6beb742a602f912b1bb64f598b1e2f3d758948721d44d468",
-                "sha256:ef1f08f2a924837e112cba2953e15aacfccbbfcd773b4b9b4723f8f2ddded08e",
-                "sha256:f411330a6376fb50e5b7a3e66894e4a39e60ca2e17dce258d53768fea06a37bd",
-                "sha256:f68996a3b3dc9335037f82754f9cdbe3a95db42bde571d8c3be26cc6245f2324",
-                "sha256:f7fdf55283ad38c33e35e2855565361f4bf0abd02470b8ab28d499c663bc5d7c",
-                "sha256:f963c6b1218b96db85fc37a9f0851eaf8b9040aa46dec112611697a7023da535",
-                "sha256:fa2818759aba55df50592ecbc95ebcdc99917fa7b55cc6796235b04193eb3c55",
-                "sha256:fae5cb554b604b3f9e2c608241b5d8d303e410d7dfb6d397c335f983495ce7f6",
-                "sha256:fb39aca7a64ad0c9490adfa719dbeeb87d13be137ca189d2564e596f8ba32c07"
+                "sha256:00215f6a9058fbf84f9d47536902558eb61f180a6b2a0fa35338d06ceb9a2e5a",
+                "sha256:0028eb0967942d0d2891eae700ae1a27b7fd18604cfcb16a1ef486a790fee99e",
+                "sha256:0155c33af0676fc38e1107679be882077680ad1abb6303956b97259c3177e85e",
+                "sha256:063411228b852fb2ed7485cf91f8e7d30893e69b0acb207ec349db04cccc8225",
+                "sha256:0700c2133ba203c4068aaecd6a59bda22e06a5e46255c9da23cbf68c6942215d",
+                "sha256:08e08ccf5b10badb7d0a5c84829b914c6e1e1f3a716fdb2bf294e2bd01562775",
+                "sha256:0d292cabd7c8335bdd3237ded442480a249dbcdb4ddfac5218799364a01a0f5c",
+                "sha256:15932ec5f224b0e35764dc156514533a4fca52dcfda0dfbe462a1a22b37efd59",
+                "sha256:18f87baa20e02e9277ad8960cd89b63c79c05caf106f4c959a9595c43f2a34a5",
+                "sha256:1a6420a36975e0073acaeee44ead260c1f6ea56812cfc6c31ec00c1c48197173",
+                "sha256:1b401e8b9aece651512e62c431181e6e83048a651698a727ea0eb0699e9f9b74",
+                "sha256:1d7b7b71bcb82d8713c7c2e9c5f061415598af5938666beded20d81fa23e7640",
+                "sha256:23750a9b8a329844ba1fe267ca456bb3184984da2880ed17ae641c5af8de3fef",
+                "sha256:23a059143c1393015c68936370cce11690f7294731904bdae47cc3e16d0b2474",
+                "sha256:26d9fd624649a10e4610fab2bc820e215a184d193e47d0be7fe53c1c8f67f370",
+                "sha256:291c9ce3929a75b45ce8ddde2aa7694fc8449f2bc8f5bd93adf021efaae2d10b",
+                "sha256:298e8b5d8087e0330aac211c85428c8761230ef46a1f2c516d6a2f67fb8803c5",
+                "sha256:2c7c4266c1b61eb429e8aeb7d8ed6a3bfe6c890a1788b18dbec090c35c6b93fa",
+                "sha256:2d68a8e8a3a816629283faf82358d8c93fe5bd974dd2704152394a3de4cec22a",
+                "sha256:344b89384c250ba6a4ce1786e04d01500e4dac0f4137ceebcaad12973c0ac0b3",
+                "sha256:3455ecc46ea443b5f7d9c2f946ce4017745e017b0d0f8b99c92564eff97e97f5",
+                "sha256:3d544a614055b131111bed6edfa1cb0fb082a7265761bcb03321f2dd7b5c6c48",
+                "sha256:3e5c26905aa651cc8c0ddc45e0e5dea2a1296f70bdc96af17aee9d0493280a17",
+                "sha256:3f5cc8c7bc99d2bbcd704cef165ca7d155cd6464c86cbda8339026a42d219397",
+                "sha256:4992266817169997854f81df7f6db7bdcda1609972d8ffd6919252f09ec3c0f6",
+                "sha256:4d55528ef13af4b4e074d067977b1f61408602f53ae4537dccf42ba665c2c7bd",
+                "sha256:576da63eae7809f375932bfcbca2cf20620a1915bf2fedce4b9cc8491eceefe3",
+                "sha256:58fc4d66ee349a23dbf08c7e964120dc9027059566e29cf0ce6205d590ed7eca",
+                "sha256:5b9bf77008f2c55dabbd099fd3ac87009471d223a1c7ebea36873d39511b780a",
+                "sha256:5e7996aed3f65667c6dcc8302a69368435a87c2364079a066750a2eac75ea01e",
+                "sha256:5f7487be65b9c2c510819e744e375bd41b929a97e5915c4852a82fbb085df62c",
+                "sha256:6388e4e95a26717b94a05ced084e19da4d92aca883f392dffcf8e48c8e221a24",
+                "sha256:65af12f70355de29e1092f319f85a3467f4005e959ab65129cb697169ce94b86",
+                "sha256:668d2b45d62c68c7a370ac3dce108ffda482b0a0f50abd8b4c604a813a59e08f",
+                "sha256:71333c22f7cf5f0480b59a0aef21f652cf9bbaa9679ad261b405b65a57511d1e",
+                "sha256:7150b83b3e3ddaac81a8bb6a9b5f93117674a0e7a2b5a5b32ab31fdfea6df27f",
+                "sha256:748e472345c3a82cfb462d0dff998a7bf43e621eed73374cb19f307e97e08a83",
+                "sha256:75dbfd41a61bc1fb0536bf7b1abf272dc115c53d4d77db770cd65d46d4520882",
+                "sha256:7618a082c55cf038eede4a918c1001cc8a4411dfe508dc762659bcd48d8f4c6e",
+                "sha256:780fcb855be29153901c67fc9c5633d48aebef21b90aa72812fa181d731c6b00",
+                "sha256:78d10c431073dc6ebceed35ab22948a016cc2b5120963c13a41e38bdde4a7212",
+                "sha256:7a3a3d3e4f1e3cd2a67b93a0b6ed0f2499e33f47cc568e3a0023e405abdc0ff1",
+                "sha256:7b6975d3763d0952c111700c0634968419268e6bbc0b55fe71138987fa66f309",
+                "sha256:80772e3bda6787510d9620bc0c7572be404a922f8ccdfd436bf6c3778119464c",
+                "sha256:80992eb20755701753e30a6952a96aa58f353d12a65ad3c9d48a8da5ec4690cf",
+                "sha256:841128a22e6ac04070a0f84776d07e9c38c4dcce8e28792a95e45fc621605517",
+                "sha256:861d25ae0985a1dd5297fee35f476b60c6029e2e6e19847d5b4d0a43a390b696",
+                "sha256:872f3dcaa8bf2245944861d7311179d2c0c9b2aaa7d3b464d99a7c2e401f01fa",
+                "sha256:87c93b25d538c433fb053da6228c6290117ba53ff6a537c133b0f2087948a582",
+                "sha256:8856aa76839dc234d3469f1e270918ce6bec1d6a601eba928f45d68a15f04fc3",
+                "sha256:885e023e73ce09b11b89ab91fc60f35d80878d2c19d6213a32b42ff36543c291",
+                "sha256:899b5e7e2d5a8bc92aa533c2d4e55e5ebba095c485568a5e4bedbc163421259a",
+                "sha256:8ce8caa29ebbdcde67e5fd652c811d34bc01f249dbc0d61e5cc4db05ae79a83b",
+                "sha256:8e1c68303ccf7fceb50fbab79064a2636119fd9aca121f28453709283dbca727",
+                "sha256:8e7e2b3577e97fa43c2c2b12a16139b2cedbd0770235d5179c0412b4794efd9b",
+                "sha256:92f05fc7d832e970047662b3440b190d24ea04f8d3c760e33e7163b67308c878",
+                "sha256:97f5811df21703446b42303475b8b855ee07d6ab6cdf8565eff115540624f25d",
+                "sha256:9affee8cb1ec453382c27eb9043378ab32f49cd4bc24a24275f5c39bf186c279",
+                "sha256:a2da4a8c6d465fde36cea7d54bf47b5cf089073452f0e47c8632ecb9dec23c07",
+                "sha256:a6903cdca64f1e301af9be424798328c1fe3b4b14aede35f04510989fc72f012",
+                "sha256:a8ab1adf04ae2d6d65835995218fd3f3eb644fe20655ca8ee233e2c7270ff53b",
+                "sha256:a8edd467551c1102dc0f5754ab55cd0703431cd3044edf8c8e7d9208d63fa453",
+                "sha256:ac00c41dd315d147b129976204839ca9de699d83519ff1272afbe4fb9d362d12",
+                "sha256:ad277f74b1c164f7248afa968700e410651eb858d7c160d109fb451dc45a2f09",
+                "sha256:ae46a50d235f1631d9ec4670503f7b30405103034830bc13df29fd947207f795",
+                "sha256:afe6b5a04b2ab1aa89bad32ca47bf71358e7302a06fdfdad857389dca8fb5f04",
+                "sha256:b1cb078f54af0abd835ca76f93a3152565b73be0f056264da45117d0adf5e99c",
+                "sha256:b25136212a3d064a8f0b9ebbb6c57094c5229e0de76d15c79b76feff26aeb7b8",
+                "sha256:b3226b246facae14909b465061ddcfa2dfeadb6a64f407f24300d42d69bcb1a1",
+                "sha256:b98e75b21fc2ba5285aef8efaf34131d16af1c38df36bdca2f50634bea2d3060",
+                "sha256:bbd7b24d108509a1b9b6679fcc1166a7dd031dbef1f3c2c73788f42e3ebb3beb",
+                "sha256:bed57543c99249ab3a4586ddc8786529fbc33309e5e8a1351802a06ca2baf4c2",
+                "sha256:c0583f69522732bdd79dca4cd3873e63a29acf4a299769c7541f2ca1e4dd4bc6",
+                "sha256:c1e0e9916301e3b3d970814b1439ca59487f0616d30f36a44cead66ee1748c31",
+                "sha256:c651847545422c8131660704c58606d841e228ed576c8f1666d98b3d318f89da",
+                "sha256:c7853f27195598e550fe089f78f0732c66ee1d1f0eaae8ad081589a5a2f5d4af",
+                "sha256:cbae50d352e4717ffc22c566afc2d0da744380e87ed44a144508e3fb9114a3f4",
+                "sha256:cdbed8f21204398f47de39b0a9b180d7e571f02dfb18bf5f1b618e238454b685",
+                "sha256:d08395595c42bcd82c3608762ce734504c6d025eef1c06f42326a6023a584186",
+                "sha256:d4639111e73997567343df6551da9dd90d66aece1b9fc26c786d328439488103",
+                "sha256:d63787f289944cc4bde518ad2b5e70a4f0d6e2ce76324635359c74c113fd188f",
+                "sha256:d6d5f061f6a2aa55790b9e64a23dfd87b6664ab56e24cd06c78eb43986cb260b",
+                "sha256:d7865df1fb564092bcf46dac61b5def25342faf6352e4bc0e61a286e3fa26a3d",
+                "sha256:db6585b600b2e76e98131e0ac0e5195759082b51687ad0c94505970c90718f4a",
+                "sha256:e36d7369363d2707d5f68950a64c4e025991eb0177db01ccb6aa6facae48b69f",
+                "sha256:e7947d9a6264c727a556541b1630296bbd5d0a05068d21c38dde8e7a1c703ef0",
+                "sha256:eb2d59bc196e6d3b1827c7db06c1a898bfa0787c0574af398e65ccf2e97c0fbe",
+                "sha256:ee9c2f6ca9774c2c24bbf7b23086264e6b5fa178201450535ec0859739e6f78d",
+                "sha256:f4760e1b02173f4155203054f77a5dc0b4078de7645c922b208d28e7eb99f3e2",
+                "sha256:f70bec8a14a692be6dbe7ce8aab303e88df891cbd4a39af091f90b6702e28055",
+                "sha256:f869e34d2326e417baee430ae998e91412cc8e7fdd83d979277a90a0e79a5b47",
+                "sha256:f8b9a7cd381970e64849070aca7c32d53ab7d96c66db6c2ef7aa23c6e803f514",
+                "sha256:f99d74ddf9d3b6126b509e81865f89bd1283e3fc1b568b68cd7bd9dfa15583d7",
+                "sha256:f9e7e493ded7042712a374471203dd43ae3fff5b81e3de1a0513fa241af9fd41",
+                "sha256:fc72ae476732cdb7b2c1acb5af23b478b8a0d4b6fcf19b90dd150291e0d5b26b",
+                "sha256:fccbf0cd3411719e4c9426755df90bf3449d9fc5a89f077f4a7f1abd4f70c910",
+                "sha256:ffcf18ad3edf1c170e27e88b10282a2c449aa0358659592462448d71b2000cfc"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.9.2"
+            "version": "==0.10.0"
         },
         "s3transfer": {
             "hashes": [
@@ -1849,50 +1842,66 @@
         },
         "safetensors": {
             "hashes": [
-                "sha256:042a60f633c3c7009fdf6a7c182b165cb7283649d2a1e9c7a4a1c23454bd9a5b",
-                "sha256:14e8c19d6dc51d4f70ee33c46aff04c8ba3f95812e74daf8036c24bc86e75cae",
-                "sha256:155b82dbe2b0ebff18cde3f76b42b6d9470296e92561ef1a282004d449fa2b4c",
-                "sha256:16063d94d8f600768d3c331b1e97964b1bf3772e19710105fe24ec5a6af63770",
-                "sha256:2961c1243fd0da46aa6a1c835305cc4595486f8ac64632a604d0eb5f2de76175",
-                "sha256:2dbd34554ed3b99435a0e84df077108f5334c8336b5ed9cb8b6b98f7b10da2f6",
-                "sha256:30a75707be5cc9686490bde14b9a371cede4af53244ea72b340cfbabfffdf58a",
-                "sha256:33409df5e28a83dc5cc5547a3ac17c0f1b13a1847b1eb3bc4b3be0df9915171e",
-                "sha256:37764b3197656ef507a266c453e909a3477dabc795962b38e3ad28226f53153b",
-                "sha256:3d7d70d48585fe8df00725aa788f2e64fd24a4c9ae07cd6be34f6859d0f89a9c",
-                "sha256:4939067736783acd8391d83cd97d6c202f94181951ce697d519f9746381b6a39",
-                "sha256:50224a1d99927ccf3b75e27c3d412f7043280431ab100b4f08aad470c37cf99a",
-                "sha256:54ad6af663e15e2b99e2ea3280981b7514485df72ba6d014dc22dae7ba6a5e6c",
-                "sha256:564f42838721925b5313ae864ba6caa6f4c80a9fbe63cf24310c3be98ab013cd",
-                "sha256:5d344e8b2681a33aafc197c90b0def3229b3317d749531c72fa6259d0caa5c8c",
-                "sha256:670d6bc3a3b377278ce2971fa7c36ebc0a35041c4ea23b9df750a39380800195",
-                "sha256:6ff59bc90cdc857f68b1023be9085fda6202bbe7f2fd67d06af8f976d6adcc10",
-                "sha256:707df34bd9b9047e97332136ad98e57028faeccdb9cfe1c3b52aba5964cc24bf",
-                "sha256:7c864cf5dcbfb608c5378f83319c60cc9c97263343b57c02756b7613cd5ab4dd",
-                "sha256:7f80af7e4ab3188daaff12d43d078da3017a90d732d38d7af4eb08b6ca2198a5",
-                "sha256:827af9478b78977248ba93e2fd97ea307fb63f463f80cef4824460f8c2542a52",
-                "sha256:87ff0024ef2e5722a79af24688ce4a430f70601d0cf712a744105ed4b8f67ba5",
-                "sha256:8969cfd9e8d904e8d3c67c989e1bd9a95e3cc8980d4f95e4dcd43c299bb94253",
-                "sha256:8b05c93da15fa911763a89281906ca333ed800ab0ef1c7ce53317aa1a2322f19",
-                "sha256:8ed77cf358abce2307f03634694e0b2a29822e322a1623e0b1aa4b41e871bf8b",
-                "sha256:9b09f27c456efa301f98681ea14b12f81f2637889f6336223ccab71e42c34541",
-                "sha256:a86428d196959619ce90197731be9391b5098b35100a7228ef4643957648f7f5",
-                "sha256:aa98f49e95f02eb750d32c4947e7d5aa43883149ebd0414920866446525b70f0",
-                "sha256:ace5d471e3d78e0d93f952707d808b5ab5eac77ddb034ceb702e602e9acf2be9",
-                "sha256:ada0fac127ff8fb04834da5c6d85a8077e6a1c9180a11251d96f8068db922a17",
-                "sha256:b6a66989075c2891d743153e8ba9ca84ee7232c8539704488f454199b8b8f84d",
-                "sha256:bc9cfb3c9ea2aec89685b4d656f9f2296f0f0d67ecf2bebf950870e3be89b3db",
-                "sha256:becc5bb85b2947eae20ed23b407ebfd5277d9a560f90381fe2c42e6c043677ba",
-                "sha256:c1f8ab41ed735c5b581f451fd15d9602ff51aa88044bfa933c5fa4b1d0c644d1",
-                "sha256:c813920482c337d1424d306e1b05824a38e3ef94303748a0a287dea7a8c4f805",
-                "sha256:caec25fedbcf73f66c9261984f07885680f71417fc173f52279276c7f8a5edd3",
-                "sha256:cbb44e140bf2aeda98d9dde669dbec15f7b77f96a9274469b91a6cf4bcc5ec3b",
-                "sha256:de3e20a388b444381bcda1a3193cce51825ddca277e4cf3ed1fe8d9b2d5722cd",
-                "sha256:e04a7cbbb3856159ab99e3adb14521544f65fcb8548cce773a1435a0f8d78d27",
-                "sha256:ec30d78f20f1235b252d59cbb9755beb35a1fde8c24c89b3c98e6a1804cfd432",
-                "sha256:f54148ac027556eb02187e9bc1556c4d916c99ca3cb34ca36a7d304d675035c1",
-                "sha256:fafd95e5ef41e8f312e2a32b7031f7b9b2a621b255f867b221f94bb2e9f51ae8"
-            ],
-            "version": "==0.3.2"
+                "sha256:0085be33b8cbcb13079b3a8e131656e05b0bc5e6970530d4c24150f7afd76d70",
+                "sha256:034717e297849dae1af0a7027a14b8647bd2e272c24106dced64d83e10d468d1",
+                "sha256:07114cec116253ca2e7230fdea30acf76828f21614afd596d7b5438a2f719bd8",
+                "sha256:08f26b61e1b0a14dc959aa9d568776bd038805f611caef1de04a80c468d4a7a4",
+                "sha256:0c345fd85b4d2093a5109596ff4cd9dfc2e84992e881b4857fbc4a93a3b89ddb",
+                "sha256:0f2b404250b3b877b11d34afcc30d80e7035714a1116a3df56acaca6b6c00096",
+                "sha256:0fa82004eae1a71e2aa29843ef99de9350e459a0fc2f65fc6ee0da9690933d2d",
+                "sha256:10d637423d98ab2e6a4ad96abf4534eb26fcaf8ca3115623e64c00759374e90d",
+                "sha256:12b83f1986cd16ea0454c636c37b11e819d60dd952c26978310a0835133480b7",
+                "sha256:178fd30b5dc73bce14a39187d948cedd0e5698e2f055b7ea16b5a96c9b17438e",
+                "sha256:17f41344d9a075f2f21b289a49a62e98baff54b5754240ba896063bce31626bf",
+                "sha256:1c32ee08f61cea56a5d62bbf94af95df6040c8ab574afffaeb7b44ae5da1e9e3",
+                "sha256:25149180d4dc8ca48bac2ac3852a9424b466e36336a39659b35b21b2116f96fc",
+                "sha256:2fff5b19a1b462c17322998b2f4b8bce43c16fe208968174d2f3a1446284ceed",
+                "sha256:351600f367badd59f7bfe86d317bb768dd8c59c1561c6fac43cafbd9c1af7827",
+                "sha256:3b46f5de8b44084aff2e480874c550c399c730c84b2e8ad1bddb062c94aa14e9",
+                "sha256:3bd2704cb41faa44d3ec23e8b97330346da0395aec87f8eaf9c9e2c086cdbf13",
+                "sha256:3dcadb6153c42addc9c625a622ebde9293fabe1973f9ef31ba10fb42c16e8536",
+                "sha256:3e3ec70c87b1e910769034206ad5efc051069b105aac1687f6edcd02526767f4",
+                "sha256:3e8fdf7407dba44587ed5e79d5de3533d242648e1f2041760b21474bd5ea5c8c",
+                "sha256:41adb1d39e8aad04b16879e3e0cbcb849315999fad73bc992091a01e379cb058",
+                "sha256:4b2951bf3f0ad63df5e6a95263652bd6c194a6eb36fd4f2d29421cd63424c883",
+                "sha256:59a596b3225c96d59af412385981f17dd95314e3fffdf359c7e3f5bb97730a19",
+                "sha256:69ccee8d05f55cdf76f7e6c87d2bdfb648c16778ef8acfd2ecc495e273e9233e",
+                "sha256:6ab43aeeb9eadbb6b460df3568a662e6f1911ecc39387f8752afcb6a7d96c087",
+                "sha256:79d1b6c7ed5596baf79c80fbce5198c3cdcc521ae6a157699f427aba1a90082d",
+                "sha256:7d3b744cee8d7a46ffa68db1a2ff1a1a432488e3f7a5a97856fe69e22139d50c",
+                "sha256:82a16e92210a6221edd75ab17acdd468dd958ef5023d9c6c1289606cc30d1479",
+                "sha256:8530399666748634bc0b301a6a5523756931b0c2680d188e743d16304afe917a",
+                "sha256:92e4d0c8b2836120fddd134474c5bda8963f322333941f8b9f643e5b24f041eb",
+                "sha256:93d54166072b143084fdcd214a080a088050c1bb1651016b55942701b31334e4",
+                "sha256:98a929e763a581f516373ef31983ed1257d2d0da912a8e05d5cd12e9e441c93a",
+                "sha256:9d741c1f1621e489ba10aa3d135b54202684f6e205df52e219d5eecd673a80c9",
+                "sha256:a002868d2e3f49bbe81bee2655a411c24fa1f8e68b703dec6629cb989d6ae42e",
+                "sha256:a07121f427e646a50d18c1be0fa1a2cbf6398624c31149cd7e6b35486d72189e",
+                "sha256:a85e29cbfddfea86453cc0f4889b4bcc6b9c155be9a60e27be479a34e199e7ef",
+                "sha256:ab29f54c6b8c301ca05fa014728996bd83aac6e21528f893aaf8945c71f42b6d",
+                "sha256:ad3cc8006e7a86ee7c88bd2813ec59cd7cc75b03e6fa4af89b9c7b235b438d68",
+                "sha256:b43956ef20e9f4f2e648818a9e7b3499edd6b753a0f5526d4f6a6826fbee8446",
+                "sha256:c0008aab36cd20e9a051a68563c6f80d40f238c2611811d7faa5a18bf3fd3984",
+                "sha256:c08a9a4b7a4ca389232fa8d097aebc20bbd4f61e477abc7065b5c18b8202dede",
+                "sha256:c31ca0d8610f57799925bf08616856b39518ab772c65093ef1516762e796fde4",
+                "sha256:c4e342fd54e66aa9512dd13e410f791e47aa4feeb5f4c9a20882c72f3d272f29",
+                "sha256:c9e943bf78c39de8865398a71818315e7d5d1af93c7b30d4da3fc852e62ad9bc",
+                "sha256:cbc3312f134baf07334dd517341a4b470b2931f090bd9284888acb7dfaf4606f",
+                "sha256:cccfcac04a010354e87c7a2fe16a1ff004fc4f6e7ef8efc966ed30122ce00bc7",
+                "sha256:d15030af39d5d30c22bcbc6d180c65405b7ea4c05b7bab14a570eac7d7d43722",
+                "sha256:d61a99b34169981f088ccfbb2c91170843efc869a0a0532f422db7211bf4f474",
+                "sha256:e13adad4a3e591378f71068d14e92343e626cf698ff805f61cdb946e684a218e",
+                "sha256:e76da691a82dfaf752854fa6d17c8eba0c8466370c5ad8cf1bfdf832d3c7ee17",
+                "sha256:eaf0e4bc91da13f21ac846a39429eb3f3b7ed06295a32321fa3eb1a59b5c70f3",
+                "sha256:edb7072d788c4f929d0f5735d3a2fb51e5a27f833587828583b7f5747af1a2b8",
+                "sha256:f1045f798e1a16a6ced98d6a42ec72936d367a2eec81dc5fade6ed54638cd7d2",
+                "sha256:f2f59fce31dd3429daca7269a6b06f65e6547a0c248f5116976c3f1e9b73f251",
+                "sha256:f439175c827c2f1bbd54df42789c5204a10983a30bc4242bc7deaf854a24f3f0",
+                "sha256:f490132383e5e490e710608f4acffcb98ed37f91b885c7217d3f9f10aaff9048",
+                "sha256:f579877d30feec9b6ba409d05fa174633a4fc095675a4a82971d831a8bb60b97",
+                "sha256:f84a74cbe9859b28e3d6d7715ac1dd3097bebf8d772694098f6d42435245860c"
+            ],
+            "version": "==0.3.3"
         },
         "scikit-learn": {
             "hashes": [
@@ -1923,28 +1932,34 @@
         },
         "scipy": {
             "hashes": [
-                "sha256:08d957ca82d3535b3b9ba6c8ff355d78fe975271874e2af267cb5add5bd78625",
-                "sha256:249cfa465c379c9bb2c20123001e151ff5e29b351cbb7f9c91587260602c58d0",
-                "sha256:366a6a937110d80dca4f63b3f5b00cc89d36f678b2d124a01067b154e692bab1",
-                "sha256:39154437654260a52871dfde852adf1b93b1d1bc5dc0ffa70068f16ec0be2624",
-                "sha256:396fae3f8c12ad14c5f3eb40499fd06a6fef8393a6baa352a652ecd51e74e029",
-                "sha256:3b9963798df1d8a52db41a6fc0e6fa65b1c60e85d73da27ae8bb754de4792481",
-                "sha256:3e8eb42db36526b130dfbc417609498a6192381abc1975b91e3eb238e0b41c1a",
-                "sha256:512fdc18c65f76dadaca139348e525646d440220d8d05f6d21965b8d4466bccd",
-                "sha256:aec8c62fbe52914f9cf28d846cf0401dd80ab80788bbab909434eb336ed07c04",
-                "sha256:b41a0f322b4eb51b078cb3441e950ad661ede490c3aca66edef66f4b37ab1877",
-                "sha256:b4bb943010203465ac81efa392e4645265077b4d9e99b66cf3ed33ae12254173",
-                "sha256:b588311875c58d1acd4ef17c983b9f1ab5391755a47c3d70b6bd503a45bfaf71",
-                "sha256:ba94eeef3c9caa4cea7b402a35bb02a5714ee1ee77eb98aca1eed4543beb0f4c",
-                "sha256:be8c962a821957fdde8c4044efdab7a140c13294997a407eaee777acf63cbf0c",
-                "sha256:cce154372f0ebe88556ed06d7b196e9c2e0c13080ecb58d0f35062dc7cc28b47",
-                "sha256:d51565560565a0307ed06fa0ec4c6f21ff094947d4844d6068ed04400c72d0c3",
-                "sha256:e866514bc2d660608447b6ba95c8900d591f2865c07cca0aa4f7ff3c4ca70f30",
-                "sha256:fb5b492fa035334fd249f0973cc79ecad8b09c604b42a127a677b45a9a3d4289",
-                "sha256:ffb28e3fa31b9c376d0fb1f74c1f13911c8c154a760312fbee87a21eb21efe31"
+                "sha256:0f3261f14b767b316d7137c66cc4f33a80ea05841b9c87ad83a726205b901423",
+                "sha256:10eb6af2f751aa3424762948e5352f707b0dece77288206f227864ddf675aca0",
+                "sha256:1342ca385c673208f32472830c10110a9dcd053cf0c4b7d4cd7026d0335a6c1d",
+                "sha256:214cdf04bbae7a54784f8431f976704ed607c4bc69ba0d5d5d6a9df84374df76",
+                "sha256:2b997a5369e2d30c97995dcb29d638701f8000d04df01b8e947f206e5d0ac788",
+                "sha256:2c91cf049ffb5575917f2a01da1da082fd24ed48120d08a6e7297dfcac771dcd",
+                "sha256:3aeb87661de987f8ec56fa6950863994cd427209158255a389fc5aea51fa7055",
+                "sha256:4447ad057d7597476f9862ecbd9285bbf13ba9d73ce25acfa4e4b11c6801b4c9",
+                "sha256:542a757e2a6ec409e71df3d8fd20127afbbacb1c07990cb23c5870c13953d899",
+                "sha256:8d9886f44ef8c9e776cb7527fb01455bf4f4a46c455c4682edc2c2cc8cd78562",
+                "sha256:90d3b1364e751d8214e325c371f0ee0dd38419268bf4888b2ae1040a6b266b2a",
+                "sha256:95763fbda1206bec41157582bea482f50eb3702c85fffcf6d24394b071c0e87a",
+                "sha256:ac74b1512d38718fb6a491c439aa7b3605b96b1ed3be6599c17d49d6c60fca18",
+                "sha256:afdb0d983f6135d50770dd979df50bf1c7f58b5b33e0eb8cf5c73c70600eae1d",
+                "sha256:b0620240ef445b5ddde52460e6bc3483b7c9c750275369379e5f609a1050911c",
+                "sha256:b133f237bd8ba73bad51bc12eb4f2d84cbec999753bf25ba58235e9fc2096d80",
+                "sha256:b29318a5e39bd200ca4381d80b065cdf3076c7d7281c5e36569e99273867f61d",
+                "sha256:b8425fa963a32936c9773ee3ce44a765d8ff67eed5f4ac81dc1e4a819a238ee9",
+                "sha256:d2b813bfbe8dec6a75164523de650bad41f4405d35b0fa24c2c28ae07fcefb20",
+                "sha256:d690e1ca993c8f7ede6d22e5637541217fc6a4d3f78b3672a6fe454dbb7eb9a7",
+                "sha256:e367904a0fec76433bf3fbf3e85bf60dae8e9e585ffd21898ab1085a29a04d16",
+                "sha256:ea932570b1c2a30edafca922345854ff2cd20d43cd9123b6dacfdecebfc1a80b",
+                "sha256:f28f1f6cfeb48339c192efc6275749b2a25a7e49c4d8369a28b6591da02fbc9a",
+                "sha256:f73102f769ee06041a3aa26b5841359b1a93cc364ce45609657751795e8f4a4a",
+                "sha256:fa4909c6c20c3d91480533cddbc0e7c6d849e7d9ded692918c76ce5964997898"
             ],
             "markers": "python_version < '3.13' and python_version >= '3.9'",
-            "version": "==1.11.1"
+            "version": "==1.11.2"
         },
         "sentencepiece": {
             "hashes": [
@@ -1999,11 +2014,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:d59c97e7b774979a5ccb96388efc9eb65518004537e85d52e81eaee89ab6dd91",
-                "sha256:e13e1b0bc760e9b0127eda042845999b2f913e12437046e663b833aa96d89715"
+                "sha256:3d4dfa6d95f1b101d695a6160a7626e15583af71a5f52176efa5d39a054d475d",
+                "sha256:3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==68.1.0"
+            "version": "==68.1.2"
         },
         "simplejson": {
             "hashes": [
@@ -2461,11 +2476,11 @@
         },
         "websocket-client": {
             "hashes": [
-                "sha256:c951af98631d24f8df89ab1019fc365f2227c0892f12fd150e935607c79dd0dd",
-                "sha256:f1f9f2ad5291f0225a49efad77abf9e700b6fef553900623060dad6e26503b9d"
+                "sha256:53e95c826bf800c4c465f50093a8c4ff091c7327023b10bfaff40cf1ef170eaa",
+                "sha256:ce54f419dfae71f4bdba69ebe65bf7f0a93fe71bc009ad3a010aacc3eebad537"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.6.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.6.2"
         },
         "xxhash": {
             "hashes": [
@@ -2685,11 +2700,11 @@
         },
         "click": {
             "hashes": [
-                "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd",
-                "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"
+                "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28",
+                "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.1.6"
+            "version": "==8.1.7"
         },
         "coverage": {
             "hashes": [
@@ -2879,11 +2894,11 @@
         },
         "pluggy": {
             "hashes": [
-                "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849",
-                "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"
+                "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12",
+                "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.2.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.3.0"
         },
         "pycodestyle": {
             "hashes": [

From 32b16a5616800868213650754ec13b0888c9ca0b Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@users.noreply.github.com>
Date: Mon, 28 Aug 2023 13:40:19 +0000
Subject: [PATCH 29/52] Update requirements.txt

---
 requirements.txt | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 5e11a9b7db..da0bce8810 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -12,20 +12,20 @@ bitsandbytes==0.41.1
 bleach==6.0.0 ; python_version >= '3.7'
 blessed==1.20.0 ; python_version >= '2.7'
 bokeh==3.2.1
-boto3==1.28.28
-botocore==1.31.28 ; python_version >= '3.7'
+boto3==1.28.35
+botocore==1.31.35 ; python_version >= '3.7'
 bravado==11.0.3 ; python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_full_version != '3.5.0'
 bravado-core==6.1.0 ; python_version >= '3.7'
 certifi==2023.7.22 ; python_version >= '3.6'
 charset-normalizer==3.2.0 ; python_full_version >= '3.7.0'
-click==8.1.6 ; python_version >= '3.7'
-cmake==3.27.2
+click==8.1.7 ; python_version >= '3.7'
+cmake==3.27.4.1
 colorama==0.4.6 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
 contourpy==1.1.0 ; python_version >= '3.8'
 coolname==2.2.0
 cramjam==2.7.0 ; python_version >= '3.7'
 datasets==2.14.4
-deepspeed==0.10.0
+deepspeed==0.10.1
 dill==0.3.7
 docopt==0.6.2
 einops==0.6.1
@@ -70,10 +70,10 @@ networkx==3.1 ; python_version >= '3.8'
 ninja==1.11.1
 numpy==1.25.2
 oauthlib==3.2.2 ; python_version >= '3.6'
-openai==0.27.8
+openai==0.27.9
 packaging==23.1 ; python_version >= '3.7'
 pandas==1.5.3
-peft==0.4.0
+peft==0.5.0
 pillow==10.0.0 ; python_version >= '3.8'
 pipreqs==0.4.13 ; python_version >= '3.7'
 portalocker==2.7.0 ; python_version >= '3.5'
@@ -95,16 +95,16 @@ requests==2.31.0 ; python_version >= '3.7'
 requests-oauthlib==1.3.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 rfc3339-validator==0.1.4
 rfc3987==1.3.8
-rpds-py==0.9.2 ; python_version >= '3.8'
+rpds-py==0.10.0 ; python_version >= '3.8'
 s3transfer==0.6.2 ; python_version >= '3.7'
 sacrebleu==2.0.0
-safetensors==0.3.2
+safetensors==0.3.3
 scikit-learn==1.3.0
-scipy==1.11.1 ; python_version < '3.13' and python_version >= '3.9'
+scipy==1.11.2 ; python_version < '3.13' and python_version >= '3.9'
 sentencepiece==0.1.99
-setuptools==68.1.0 ; python_version >= '3.8'
-simplejson==3.19.1 ; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'
-six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+setuptools==68.1.2 ; python_version >= '3.8'
+simplejson==3.19.1 ; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2'
+six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
 smmap==5.0.0 ; python_version >= '3.6'
 sniffio==1.3.0 ; python_version >= '3.7'
 soupsieve==2.4.1 ; python_version >= '3.7'
@@ -133,7 +133,7 @@ uvicorn==0.23.2 ; python_version >= '3.8'
 wcwidth==0.2.6
 webcolors==1.13
 webencodings==0.5.1
-websocket-client==1.6.1 ; python_version >= '3.7'
+websocket-client==1.6.2 ; python_version >= '3.8'
 xxhash==3.3.0 ; python_version >= '3.7'
 xyzservices==2023.7.0 ; python_version >= '3.8'
 yarg==0.1.9

From e36fada0fe783c516662feeebd029f294f41bf01 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Mon, 28 Aug 2023 21:31:17 -0700
Subject: [PATCH 30/52] make black

---
 .../python_configs/text_causal_language_modeling_config.py  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llm_studio/python_configs/text_causal_language_modeling_config.py b/llm_studio/python_configs/text_causal_language_modeling_config.py
index 62bd361382..c335cc7b87 100644
--- a/llm_studio/python_configs/text_causal_language_modeling_config.py
+++ b/llm_studio/python_configs/text_causal_language_modeling_config.py
@@ -347,9 +347,9 @@ class ConfigNLPCausalLMEnvironment(DefaultConfig):
     deepspeed_reduce_bucket_size: int = 1e6
     deepspeed_stage3_prefetch_bucket_size: int = 1e6
     deepspeed_stage3_param_persistence_threshold: int = 1e6
-#     deepspeed_offload_optimizer: bool = False
-#     deepspeed_stage3_max_live_parameters: int = 1e9
-#     deepspeed_stage3_max_reuse_distance: int = 1e9
+    #     deepspeed_offload_optimizer: bool = False
+    #     deepspeed_stage3_max_live_parameters: int = 1e9
+    #     deepspeed_stage3_max_reuse_distance: int = 1e9
 
     find_unused_parameters: bool = False
     trust_remote_code: bool = True

From b5e59e91b0d94a51851fd1b8724c1f248068c7f4 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Mon, 28 Aug 2023 21:44:49 -0700
Subject: [PATCH 31/52] add default

---
 llm_studio/src/utils/modeling_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index 137d88e4b3..814c777971 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -54,7 +54,7 @@ def unwrap_model(model: torch.nn.Module):
     return model
 
 
-def check_disk_space(model: torch.nn.Module, path: str, use_deepspeed: bool):
+def check_disk_space(model: torch.nn.Module, path: str, use_deepspeed: bool = False):
     total, used, free = shutil.disk_usage(path)
 
     model_size_in_bytes = 0

From 24eeb1627b4c5a84cb1c5e96ee15817226071f1c Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Mon, 4 Sep 2023 06:47:00 -0700
Subject: [PATCH 32/52] minor fix

---
 llm_studio/app_utils/utils.py | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/llm_studio/app_utils/utils.py b/llm_studio/app_utils/utils.py
index f917b1143b..a3b8bfecf9 100644
--- a/llm_studio/app_utils/utils.py
+++ b/llm_studio/app_utils/utils.py
@@ -142,20 +142,22 @@ def start_process(
         free_port = find_free_port()
         if cfg.environment.use_deepspeed:
             logger.info("Starting deepspeed...")
+            cmd = [
+                "env",
+                "deepspeed",
+                "--include",
+                f"localhost:{','.join(gpu_list)}",
+                "--master_port",
+                f"{str(free_port)}",
+                "train_wave.py",
+                "-Y",
+                config_name,
+            ]
+            if len(process_queue) > 0:
+                cmd.append("-Q")
+                cmd.append(",".join([str(x) for x in process_queue]))
             p = subprocess.Popen(
-                [
-                    "env",
-                    "deepspeed",
-                    "--include",
-                    f"localhost:{','.join(gpu_list)}",
-                    "--master_port",
-                    f"{str(free_port)}",
-                    "train_wave.py",
-                    "-Y",
-                    config_name,
-                    "-Q",
-                    ",".join([str(x) for x in process_queue]),
-                ],
+                cmd,
                 env=env,
             )
         else:

From b9e59345455a02648ce9fc315eab6a3c3df724a7 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Mon, 4 Sep 2023 06:51:00 -0700
Subject: [PATCH 33/52] minor fix

---
 llm_studio/app_utils/utils.py | 41 ++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 22 deletions(-)

diff --git a/llm_studio/app_utils/utils.py b/llm_studio/app_utils/utils.py
index a3b8bfecf9..43186b443d 100644
--- a/llm_studio/app_utils/utils.py
+++ b/llm_studio/app_utils/utils.py
@@ -153,30 +153,27 @@ def start_process(
                 "-Y",
                 config_name,
             ]
-            if len(process_queue) > 0:
-                cmd.append("-Q")
-                cmd.append(",".join([str(x) for x in process_queue]))
-            p = subprocess.Popen(
-                cmd,
-                env=env,
-            )
         else:
             logger.info("Starting torchrun...")
-            p = subprocess.Popen(
-                [
-                    "env",
-                    f"CUDA_VISIBLE_DEVICES={','.join(gpu_list)}",
-                    "torchrun",
-                    f"--nproc_per_node={str(num_gpus)}",
-                    f"--master_port={str(free_port)}",
-                    "train_wave.py",
-                    "-Y",
-                    config_name,
-                    "-Q",
-                    ",".join([str(x) for x in process_queue]),
-                ],
-                env=env,
-            )
+            cmd = [
+                "env",
+                f"CUDA_VISIBLE_DEVICES={','.join(gpu_list)}",
+                "torchrun",
+                f"--nproc_per_node={str(num_gpus)}",
+                f"--master_port={str(free_port)}",
+                "train_wave.py",
+                "-Y",
+                config_name,
+            ]
+
+        if len(process_queue) > 0:
+            cmd.append("-Q")
+            cmd.append(",".join([str(x) for x in process_queue]))
+        p = subprocess.Popen(
+            cmd,
+            env=env,
+        )
+
     logger.info(f"Percentage of RAM memory used: {psutil.virtual_memory().percent}")
 
     return p

From a296cca4fdbc7b5841c40e7f783e75e45f0056dd Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Mon, 4 Sep 2023 06:55:39 -0700
Subject: [PATCH 34/52] minor fix

---
 llm_studio/app_utils/utils.py | 54 +++++++++++++++--------------------
 1 file changed, 23 insertions(+), 31 deletions(-)

diff --git a/llm_studio/app_utils/utils.py b/llm_studio/app_utils/utils.py
index 43186b443d..dad8018f3e 100644
--- a/llm_studio/app_utils/utils.py
+++ b/llm_studio/app_utils/utils.py
@@ -112,32 +112,23 @@ def start_process(
     env = {**os.environ, **env_vars}
 
     if num_gpus == 0:
-        p = subprocess.Popen(
-            [
-                "python",
-                "train_wave.py",
-                "-Y",
-                config_name,
-                "-Q",
-                ",".join([str(x) for x in process_queue]),
-            ],
-            env=env,
-        )
+        cmd = [
+            "python",
+            "train_wave.py",
+            "-Y",
+            config_name,
+        ]
     # Do not delete for debug purposes
     # elif num_gpus == 1:
-    #     p = subprocess.Popen(
-    #         [
-    #             "env",
-    #             f"CUDA_VISIBLE_DEVICES={','.join(gpu_list)}",
-    #             "python",
-    #             "-u",
-    #             "train_wave.py",
-    #             "-P",
-    #             config_name,
-    #             "-Q",
-    #             ",".join([str(x) for x in process_queue]),
-    #         ]
-    #     )
+    #     cmd = [
+    #         "env",
+    #         f"CUDA_VISIBLE_DEVICES={','.join(gpu_list)}",
+    #         "python",
+    #         "-u",
+    #         "train_wave.py",
+    #         "-P",
+    #         config_name,
+    #     ]
     else:
         free_port = find_free_port()
         if cfg.environment.use_deepspeed:
@@ -166,13 +157,14 @@ def start_process(
                 config_name,
             ]
 
-        if len(process_queue) > 0:
-            cmd.append("-Q")
-            cmd.append(",".join([str(x) for x in process_queue]))
-        p = subprocess.Popen(
-            cmd,
-            env=env,
-        )
+    if len(process_queue) > 0:
+        cmd.append("-Q")
+        cmd.append(",".join([str(x) for x in process_queue]))
+
+    p = subprocess.Popen(
+        cmd,
+        env=env,
+    )
 
     logger.info(f"Percentage of RAM memory used: {psutil.virtual_memory().percent}")
 

From 11a4b8d7337162aca55441c1dccf949e2852d919 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Tue, 5 Sep 2023 01:51:45 -0700
Subject: [PATCH 35/52] fix val loader

---
 llm_studio/src/utils/modeling_utils.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index 814c777971..be2c05a43a 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -10,6 +10,7 @@
 import deepspeed
 import numpy as np
 import torch
+from deepspeed.runtime.dataloader import DeepSpeedDataLoader
 from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
 from peft import LoraConfig, get_peft_model
 from peft.utils import TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING
@@ -19,6 +20,7 @@
     MixedPrecision,
 )
 from torch.nn.parallel import DistributedDataParallel
+from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm
 from transformers import (
     AutoConfig,
@@ -288,10 +290,13 @@ def wrap_model_distributed(
             training_data=train_dataloader.dataset,
             config_params=ds_config,
         )
-        _, _, val_dataloader, _ = deepspeed.initialize(
-            model=torch.nn.Linear(1, 1),
-            training_data=val_dataloader.dataset,
-            config_params=ds_config,
+        val_dataloader = DeepSpeedDataLoader(
+            val_dataloader.dataset,
+            batch_size=val_dataloader.batch_size,
+            local_rank=cfg.environment._local_rank,
+            pin_memory=True,
+            tput_timer=None,
+            data_sampler=DistributedSampler(val_dataloader.dataset, shuffle=False),
         )
     else:
         find_unused_parameters = cfg.environment.find_unused_parameters

From 3efa2c96f5e9278ff50bc2176f2feb2b772ca4e0 Mon Sep 17 00:00:00 2001
From: Philipp Singer <killver@gmail.com>
Date: Thu, 7 Sep 2023 09:26:58 +0000
Subject: [PATCH 36/52] potential val loader fix

---
 llm_studio/src/utils/modeling_utils.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index be2c05a43a..d341836f5f 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -36,6 +36,7 @@
 from llm_studio.src.optimizers import Optimizers
 from llm_studio.src.schedulers import Schedulers
 from llm_studio.src.utils.data_utils import (
+    OrderedDistributedSampler,
     batch_padding,
     cat_batches,
     get_inference_batch_size,
@@ -296,7 +297,11 @@ def wrap_model_distributed(
             local_rank=cfg.environment._local_rank,
             pin_memory=True,
             tput_timer=None,
-            data_sampler=DistributedSampler(val_dataloader.dataset, shuffle=False),
+            data_sampler=OrderedDistributedSampler(
+                val_dataloader.dataset,
+                num_replicas=cfg.environment._world_size,
+                rank=cfg.environment._local_rank,
+            )
         )
     else:
         find_unused_parameters = cfg.environment.find_unused_parameters

From 14bc17e0642c1e12a105e357ef702bddf40b500a Mon Sep 17 00:00:00 2001
From: Philipp Singer <killver@gmail.com>
Date: Fri, 8 Sep 2023 12:25:00 +0000
Subject: [PATCH 37/52] update

---
 Pipfile                                |   2 +-
 Pipfile.lock                           | 470 +++++++++++++++----------
 llm_studio/src/utils/modeling_utils.py |   8 +-
 train.py                               |  17 +-
 4 files changed, 299 insertions(+), 198 deletions(-)

diff --git a/Pipfile b/Pipfile
index 6d92bd106c..27643d40b7 100644
--- a/Pipfile
+++ b/Pipfile
@@ -49,7 +49,7 @@ h2o-wave = "0.26"
 tiktoken = "0.4.0"
 hf-transfer = "0.1.3"
 peft = "0.4.0"
-deepspeed = ">=0.10.0"
+deepspeed = "0.10.2"
 
 [dev-packages]
 black = "==23.7.0"
diff --git a/Pipfile.lock b/Pipfile.lock
index fce8d3c754..474772133c 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "9d4761bf1f6e90b16795f2be0e8c206d12da932a246027cd39c4ccd432997bf5"
+            "sha256": "182a3e0ba04f22befc124f3ca3da39dc482220047cd52fb40b45180eaad26e08"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -132,11 +132,11 @@
         },
         "anyio": {
             "hashes": [
-                "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780",
-                "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5"
+                "sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f",
+                "sha256:f7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==3.7.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==4.0.0"
         },
         "arrow": {
             "hashes": [
@@ -204,19 +204,19 @@
         },
         "boto3": {
             "hashes": [
-                "sha256:580b584e36967155abed7cc9b088b3bd784e8242ae4d8841f58cb50ab05520dc",
-                "sha256:d77415f22bbc14f3d72eaed2fc9f96d161f3ba7686922ad26d6bbc9d4985f3df"
+                "sha256:4cd3e96900fb50bddc9f48007176c80d15396d08c5248b25a41220f3570e014f",
+                "sha256:c0211a3e830432851c73fa1e136b14dbb6d02b5c9a5e1272c557e63538620b88"
             ],
             "index": "pypi",
-            "version": "==1.28.35"
+            "version": "==1.28.43"
         },
         "botocore": {
             "hashes": [
-                "sha256:7e4534325262f43293a9cc9937cb3f1711365244ffde8b925a6ee862bcf30a83",
-                "sha256:943e1465aad66db4933b06809134bd08c5b05e8eb18c19742ffec82f54769457"
+                "sha256:b4a3a1fcf75011351e2b0d3eb991f51f8d44a375d3e065f907dac67db232fc97",
+                "sha256:d8b0c41c8c75d82f15fee57f7d54a852a99810faacbeb9d6f3f022558a2c330e"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.31.35"
+            "version": "==1.31.43"
         },
         "bravado": {
             "hashes": [
@@ -333,14 +333,21 @@
         },
         "cmake": {
             "hashes": [
+                "sha256:0fb68660ce3954de99d1f41bedcf87063325c4cc891003f12de36472fa1efa28",
+                "sha256:1aca07fccfa042a0379bb027e30f090a8239b18fd3f959391c5d77c22dd0a809",
                 "sha256:2b0b53ec2e45cfe9982d0adf833b3519efc328c1e3cffae4d237841a1ed6edf4",
                 "sha256:2f217fb281b068696fdcc4b198de62e9ded8bc0a93877684afc59db3507ccb44",
+                "sha256:315eb37233e2d0b8fa01580e33439eaeaef65f1e41ad9ca269cbe68cc0a039a4",
                 "sha256:37e9cad75184fbefe837311528d026901278b606707e9d14b58e3767d49d0aa6",
                 "sha256:4a1d22ee72dcdc32d0f8bbf5691d2e9367585db8bfeafe7cffa2c4274127a801",
                 "sha256:65b79f1e8b6fa254697ee0b411aa4dff0d2309c1405af3448adf06cbd7ef0ac5",
+                "sha256:70526bbff5eeb7d4d6b921af1b80d2d29828302882f94a2cba93ad7d469b90f6",
+                "sha256:8249a1ba7901b53661b44e59bdf6fd1e977e10843795788efe25d3374de6ed95",
                 "sha256:871c8b5eaac959f079c2389c7a7f198fa5f86a029e8726fcb1f3e13d030c33e9",
+                "sha256:94fec5a8bae1f3b62d8a8653ebcb7fa4007e2d0e713f94e4b2089f708c13548f",
                 "sha256:a3bd8b9d0e294bd2b3ce27a850c9d924aee7e4f4c0bb56d66641cc1544314f58",
                 "sha256:a504815bcba0ece9aafb48a6b7770d6479756fda92f8b62f9ab7ff8a403a12d5",
+                "sha256:b72db11e13eafb46b9c53797d141e89886293db768feabef4b841accf666de54",
                 "sha256:c36eb106dec60198264b25d4bd23cd9ea30b0af9200a143ec1db887c095306f7",
                 "sha256:ec4a5bc2376dfc57065bfde6806183b331165f33457b7cc0fc0511260dde7c72"
             ],
@@ -474,18 +481,18 @@
         },
         "datasets": {
             "hashes": [
-                "sha256:29336bd316a7d827ccd4da2236596279b20ca2ac78f64c04c9483da7cbc2459b",
-                "sha256:ef29c2b5841de488cd343cfc26ab979bff77efa4d2285af51f1ad7db5c46a83b"
+                "sha256:b738a86540ab8e1a7806c8a3790b67be0056318d0c5d5a58a1b0dbdd76c0f568",
+                "sha256:dd4155091034cba04d5a28711f2ed3944275ed15c5d0c5a2d0b6b9ea34a2bdfe"
             ],
             "index": "pypi",
-            "version": "==2.14.4"
+            "version": "==2.14.5"
         },
         "deepspeed": {
             "hashes": [
-                "sha256:6db8f6be222dcdcefa3cae1fdf80aa6512d8a6851d53f3c4232083c70aada798"
+                "sha256:286bb1f01ea7340c9e18d78a562393dd1c12cc1406f1f5996f4be1d6984360ef"
             ],
             "index": "pypi",
-            "version": "==0.10.1"
+            "version": "==0.10.2"
         },
         "dill": {
             "hashes": [
@@ -519,50 +526,50 @@
         },
         "fastparquet": {
             "hashes": [
-                "sha256:079637c50fd03c0724b7858542b5d45de80105f9fa89d8475ba73bba5c03e240",
-                "sha256:07f3a5e83b454a47b13c0b4fb2acfe862619915f3702dbc1724890e5d323abbe",
-                "sha256:0b8c9667b6a8f9835c018e481fcff9ca34661818bf02a17e13de9a86328f3780",
-                "sha256:0fdb44543f8884f7d839fcc5d65e17ded933b5b84d39e5fa0daa17982f833fa2",
-                "sha256:11d39457c5aa731f710e9fdfab6552c8399ff8f784b8ef938ea6f5dac3dcc571",
-                "sha256:19f74093ee457b9b8ca0050f3e1d13f1fa41604d10da84869b3a93ed4983b992",
-                "sha256:28a4758d83c4b60c5a0ce6e4cd438576f4d53e2776a6c0160d140c65054c5c90",
-                "sha256:28f54fdd5a3d2470834697a3f0d3cfddef51ea6c08e058f94fdb0f1d0158ead1",
-                "sha256:297e75946d817755cab3dc91b7493b03932e604d7c118ec56c2e589da1d5132c",
-                "sha256:2ed33312a2d5d1070277d8df64a20ad14cc51466172538dd4612fd9c06957685",
-                "sha256:37865fb74f3ee9b4d4c2cb4197587657ffa10e120747db5192af8d00794e3408",
-                "sha256:3d47918ca184ca39690d6903695e691aef26ed4f4d76ce7f2e4d23f409fe17ce",
-                "sha256:463190b44f545be05d31edabea99fb3ccf6a849500a0b7cf23ede1c2f4108520",
-                "sha256:4cc2caf2160f5ebbf3d83d9e9a7a5a4c3053baf6fe0bf9b7ff15672d09c017a3",
-                "sha256:4e32585eaae9a1478337aca644634f9aec33a32047b707754c9d998314dfef90",
-                "sha256:58ea7d05bec6c010ff8f0139806c152925d1d2609eb19517568714911d8cde37",
-                "sha256:66ee6448fd8750d7028ccea2072f404ab05e4e00d98fd33f02dad1260e819aaf",
-                "sha256:7ecbe7dd4f9e97d777398b63b5e04fc4424928372a1f14f38956ad0d3ba89cdf",
-                "sha256:804fb946f1cd19686d2ea34e27cb6ba679fa76e9279694d37a676747f140492f",
-                "sha256:915af569d1dd6c116e7487dfbd9b74ce3fa0f9cf896df84e646cbd945f19b51c",
-                "sha256:969f103567a63d7108b02fd11d97a80942e01bcdb7cffc3991f84aa900ed1183",
-                "sha256:a8dc1c27000a57d03975ff0801a43203efb13b8cdec570ae05235a717f85b51d",
-                "sha256:ac0d0ad6f70e170339a55e2611946e48b34519f0421abc00c667b0e564910a8a",
-                "sha256:be5dfa7f221e2bd6c32f010fba2f873e3060b109bc5b5f99bf0013a4cb36994d",
-                "sha256:c5aabbb55d35f11e2f72f18b0313aac839c0d42b1f16aa3cd22c17a1fab78ca9",
-                "sha256:cd09a57bb5169d08b1180b1611431340697aa35f2aa6bf8826ebaf81d5c95688",
-                "sha256:d24b79ba87561554abd589ad45b42199112b4f9fa56824164ffe9e3db9d0654f",
-                "sha256:de0a1250e9b0044ca218c0e5c4fcacf3eb2a5b22af2021fcff895463261280e6",
-                "sha256:de5f864f25983f04d0bdfb74c1c857f5489cb3b26bb258b3e68fad2923b4264c",
-                "sha256:e88f804f4ce6ecf89f7c034beffcbcf7fc2bcc94eb0b43e941a923e9ac4fd9cd",
-                "sha256:eed1dcdbb56c166cdfcf65bfb5d4d6abdb50d873541f8ce74d3319349c68de42",
-                "sha256:f1325922db29a065f84118ffa2976783b53e6da87680762e6dcc20885cd94082",
-                "sha256:fe517e5a1a8a7cf1cfd09a1c584c95db12de2fcfc52e8ff28e8e6705167752ed"
+                "sha256:0c1bed521f149408101e90484e21f74011caa950b3fda2674539d7295f20e5eb",
+                "sha256:12e0fecff7cbf5031944f119c6e11d8a2e578ceb1c2898aa481ddc6d1bc94b6b",
+                "sha256:160ee84241f106a21cf4b0821e95235a47cdd0e1a41c6721380c077af01d07cc",
+                "sha256:1efa5a140b77ec60fec5d41135cd06ffe6f72b229a971d3756255aeca3c2fa32",
+                "sha256:23a5b302c1cdbec8450b3ba66db44210dd6b7bbbd298a3a3482588903055347f",
+                "sha256:27be564460e391313ea6c9c0225710abd5c875fa85cab5b89f223961e3065772",
+                "sha256:2ff39bc20869829ea8fd85bf0336c5006d0ffcc276d0de070030cf8ffd929160",
+                "sha256:31ecf82b3969203ef6aea9458ae9e27387eab80ab9d590031e5bfdff96bad2f1",
+                "sha256:3619bb0fb3760038a47fd742ff66acf5b8b19f1a7b9744cb13e4dda43c314e92",
+                "sha256:3c3a1cba8c7176690ad7db242f190c9ec6d0ae0fa1d30196db76eabe35536252",
+                "sha256:4f8a1381b3272c73f85dfedbdfd7c8538b46382eaaf51f18ca0e0a9afda12cf0",
+                "sha256:6648ac0946264159dfadca8ff044d5c12cf0e5587fa2cef1709c4abf84866c06",
+                "sha256:72a34011217e7c3e5482d03cbee2687732c3b956120fa34666d771e7f5eed4fe",
+                "sha256:7784c2ab4cf7fff28adb2c6b7e88cd90c345f6e4cc1e39aead8b57bc4d001700",
+                "sha256:8697a663e1997266bda5cca8107821739bd91a591229d52e3c02ccdc50377577",
+                "sha256:8cea82b9260907219ef55db470c891a61bb75f913ccff3f0a9718ab45ceb1856",
+                "sha256:8dadde182a582313aa8976afc620d73d2f1d995f29d418ecb5fc05dd015e646c",
+                "sha256:8eb212e0379cbc01adf904a87868dcfbb5658f71816df812ea49abb42f0d7220",
+                "sha256:9814e320fe7040ea440e7fff6aa62cec5e7f0440d9047c7b5316d761f129e956",
+                "sha256:9c78407941103c8f72270027a287db7e757431f766ce3890d4c16d4773006bd8",
+                "sha256:a4f7dcd88fc7d58bdefac6b29441766ec3510cf72d4da58354f75d583bd3020d",
+                "sha256:ac6a3a1e157fcb0a89a66d593494f1849d8004a7979bb24aa5b7c01ffb4c5e66",
+                "sha256:afe08f3f5b7e8db8a81d765f05a85e05f808118fc981e87edfbc06456c1609b0",
+                "sha256:b0f233bd35e2121785b49b461430bb2448ef01a8b8f4196d14580827c24dea67",
+                "sha256:b56f048b3b89400e5781093b6160cad8acdeb15f10bc8a0c14d29f96dd5abc56",
+                "sha256:c06bdf64512807714d25760c096ea4ac8be5f3dcee6d346a3c7e34ea9622b118",
+                "sha256:c23425bd3934971075331113c974929d1362b381bb388825d5a2bf8150568806",
+                "sha256:c4b76d047511949902952d7bb22ab12129efa7ed1ddb30d07bdb087d78914079",
+                "sha256:cf33f1c9c6f02c94874fa231e28845078a413943684952c19c5b8742be0d1ef0",
+                "sha256:cfe2a49cd8ca612bfbe4a23244d4a25525468b14cc72eb4550d698ec420ed17b",
+                "sha256:dcabba07d2ad07bf07a0d85f3dcc19576c1d5c9bdd781aaa0e8673a4c8552d24",
+                "sha256:e11d951450fd0734b9ee1971123c6743f5f44296fbb7258f8c92f722c5878fde",
+                "sha256:e374e648bc93d1b499ca29cd16ea09bc7c61b6c60910377b2bbe3d6c4f089b22"
             ],
             "index": "pypi",
-            "version": "==2023.7.0"
+            "version": "==2023.8.0"
         },
         "filelock": {
             "hashes": [
-                "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81",
-                "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec"
+                "sha256:0ecc1dd2ec4672a10c8550a8182f1bd0c0a5088470ecd5a125e45f49472fac3d",
+                "sha256:f067e40ccc40f2b48395a80fcbd4728262fab54e232e090a4063ab804179efeb"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==3.12.2"
+            "markers": "python_version >= '3.8'",
+            "version": "==3.12.3"
         },
         "fqdn": {
             "hashes": [
@@ -639,6 +646,9 @@
             "version": "==1.4.0"
         },
         "fsspec": {
+            "extras": [
+                "http"
+            ],
             "hashes": [
                 "sha256:1cbad1faef3e391fba6dc005ae9b5bdcbf43005c9167ce78c915549c352c869a",
                 "sha256:d0b2f935446169753e7a5c5c55681c54ea91996cc67be93c39a154fb3a2742af"
@@ -650,7 +660,7 @@
             "hashes": [
                 "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307"
             ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'",
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==0.18.3"
         },
         "gitdb": {
@@ -663,11 +673,11 @@
         },
         "gitpython": {
             "hashes": [
-                "sha256:8d9b8cb1e80b9735e8717c9362079d3ce4c6e5ddeebedd0361b228c3a67a62f6",
-                "sha256:e3d59b1c2c6ebb9dfa7a184daf3b6dd4914237e7488a1730a6d8f6f5d0b4187f"
+                "sha256:9cbefbd1789a5fe9bcf621bb34d3f441f3a90c8461d377f84eda73e721d9b06b",
+                "sha256:c19b4292d7a1d3c0f653858db273ff8a6614100d1eb1528b014ec97286193c09"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.1.32"
+            "version": "==3.1.35"
         },
         "gputil": {
             "hashes": [
@@ -680,6 +690,7 @@
             "hashes": [
                 "sha256:03a8f4f3430c3b3ff8d10a2a86028c660355ab637cee9333d63d66b56f09d52a",
                 "sha256:0bf60faf0bc2468089bdc5edd10555bab6e85152191df713e2ab1fcc86382b5a",
+                "sha256:1087300cf9700bbf455b1b97e24db18f2f77b55302a68272c56209d5587c12d1",
                 "sha256:18a7f18b82b52ee85322d7a7874e676f34ab319b9f8cce5de06067384aa8ff43",
                 "sha256:18e98fb3de7dba1c0a852731c3070cf022d14f0d68b4c87a19cc1016f3bb8b33",
                 "sha256:1a819eef4b0e0b96bb0d98d797bef17dc1b4a10e8d7446be32d1da33e095dbb8",
@@ -705,6 +716,7 @@
                 "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91",
                 "sha256:7cafd1208fdbe93b67c7086876f061f660cfddc44f404279c1585bbf3cdc64c5",
                 "sha256:7efde645ca1cc441d6dc4b48c0f7101e8d86b54c8530141b09fd31cef5149ec9",
+                "sha256:8512a0c38cfd4e66a858ddd1b17705587900dd760c6003998e9472b77b56d417",
                 "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8",
                 "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b",
                 "sha256:910841381caba4f744a44bf81bfd573c94e10b3045ee00de0cbf436fe50673a6",
@@ -728,8 +740,10 @@
                 "sha256:c9c59a2120b55788e800d82dfa99b9e156ff8f2227f07c5e3012a45a399620b7",
                 "sha256:cd021c754b162c0fb55ad5d6b9d960db667faad0fa2ff25bb6e1301b0b6e6a75",
                 "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae",
+                "sha256:d4606a527e30548153be1a9f155f4e283d109ffba663a15856089fb55f933e47",
                 "sha256:d5508f0b173e6aa47273bdc0a0b5ba055b59662ba7c7ee5119528f466585526b",
                 "sha256:d75209eed723105f9596807495d58d10b3470fa6732dd6756595e89925ce2470",
+                "sha256:d967650d3f56af314b72df7089d96cda1083a7fc2da05b375d2bc48c82ab3f3c",
                 "sha256:db1a39669102a1d8d12b57de2bb7e2ec9066a6f2b3da35ae511ff93b01b5d564",
                 "sha256:dbfcfc0218093a19c252ca8eb9aee3d29cfdcb586df21049b9d777fd32c14fd9",
                 "sha256:e0f72c9ddb8cd28532185f54cc1453f2c16fb417a08b53a855c4e6a418edd099",
@@ -739,7 +753,7 @@
                 "sha256:f82d4d717d8ef19188687aa32b8363e96062911e63ba22a0cff7802a8e58e5f1",
                 "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"
             ],
-            "markers": "python_version >= '3' and (platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32'))))))",
+            "markers": "python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))",
             "version": "==2.0.2"
         },
         "h11": {
@@ -915,8 +929,11 @@
                 "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e",
                 "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431",
                 "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686",
+                "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c",
                 "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559",
                 "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc",
+                "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb",
+                "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939",
                 "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c",
                 "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0",
                 "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4",
@@ -924,6 +941,7 @@
                 "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575",
                 "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba",
                 "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d",
+                "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd",
                 "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3",
                 "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00",
                 "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155",
@@ -932,6 +950,7 @@
                 "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f",
                 "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8",
                 "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b",
+                "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007",
                 "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24",
                 "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea",
                 "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198",
@@ -939,9 +958,12 @@
                 "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee",
                 "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be",
                 "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2",
+                "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1",
                 "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707",
                 "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6",
+                "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c",
                 "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58",
+                "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823",
                 "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779",
                 "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636",
                 "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c",
@@ -960,7 +982,9 @@
                 "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9",
                 "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57",
                 "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc",
-                "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"
+                "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc",
+                "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2",
+                "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"
             ],
             "markers": "python_version >= '3.7'",
             "version": "==2.1.3"
@@ -1228,11 +1252,11 @@
         },
         "openai": {
             "hashes": [
-                "sha256:6a3cf8e276d1a6262b50562fbc0cba7967cfebb78ed827d375986b48fdad6475",
-                "sha256:b687761c82f5ebb6f61efc791b2083d2d068277b94802d4d1369efe39851813d"
+                "sha256:60e09edf7100080283688748c6803b7b3b52d5a55d21890f3815292a0552d83b",
+                "sha256:beabd1757e3286fa166dde3b70ebb5ad8081af046876b47c14c41e203ed22a14"
             ],
             "index": "pypi",
-            "version": "==0.27.9"
+            "version": "==0.27.10"
         },
         "packaging": {
             "hashes": [
@@ -1507,7 +1531,7 @@
                 "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
                 "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.8.2"
         },
         "python-editor": {
@@ -1530,14 +1554,16 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588",
-                "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"
+                "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b",
+                "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"
             ],
-            "version": "==2023.3"
+            "version": "==2023.3.post1"
         },
         "pyyaml": {
             "hashes": [
+                "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5",
                 "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc",
+                "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df",
                 "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741",
                 "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206",
                 "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27",
@@ -1545,7 +1571,10 @@
                 "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62",
                 "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98",
                 "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696",
+                "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290",
+                "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9",
                 "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d",
+                "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6",
                 "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867",
                 "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47",
                 "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486",
@@ -1553,9 +1582,12 @@
                 "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3",
                 "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007",
                 "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938",
+                "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0",
                 "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c",
                 "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735",
                 "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d",
+                "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28",
+                "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4",
                 "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba",
                 "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8",
                 "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5",
@@ -1570,7 +1602,9 @@
                 "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43",
                 "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859",
                 "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673",
+                "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54",
                 "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a",
+                "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b",
                 "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab",
                 "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa",
                 "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c",
@@ -1723,106 +1757,106 @@
         },
         "rpds-py": {
             "hashes": [
-                "sha256:00215f6a9058fbf84f9d47536902558eb61f180a6b2a0fa35338d06ceb9a2e5a",
-                "sha256:0028eb0967942d0d2891eae700ae1a27b7fd18604cfcb16a1ef486a790fee99e",
-                "sha256:0155c33af0676fc38e1107679be882077680ad1abb6303956b97259c3177e85e",
-                "sha256:063411228b852fb2ed7485cf91f8e7d30893e69b0acb207ec349db04cccc8225",
-                "sha256:0700c2133ba203c4068aaecd6a59bda22e06a5e46255c9da23cbf68c6942215d",
-                "sha256:08e08ccf5b10badb7d0a5c84829b914c6e1e1f3a716fdb2bf294e2bd01562775",
-                "sha256:0d292cabd7c8335bdd3237ded442480a249dbcdb4ddfac5218799364a01a0f5c",
-                "sha256:15932ec5f224b0e35764dc156514533a4fca52dcfda0dfbe462a1a22b37efd59",
-                "sha256:18f87baa20e02e9277ad8960cd89b63c79c05caf106f4c959a9595c43f2a34a5",
-                "sha256:1a6420a36975e0073acaeee44ead260c1f6ea56812cfc6c31ec00c1c48197173",
-                "sha256:1b401e8b9aece651512e62c431181e6e83048a651698a727ea0eb0699e9f9b74",
-                "sha256:1d7b7b71bcb82d8713c7c2e9c5f061415598af5938666beded20d81fa23e7640",
-                "sha256:23750a9b8a329844ba1fe267ca456bb3184984da2880ed17ae641c5af8de3fef",
-                "sha256:23a059143c1393015c68936370cce11690f7294731904bdae47cc3e16d0b2474",
-                "sha256:26d9fd624649a10e4610fab2bc820e215a184d193e47d0be7fe53c1c8f67f370",
-                "sha256:291c9ce3929a75b45ce8ddde2aa7694fc8449f2bc8f5bd93adf021efaae2d10b",
-                "sha256:298e8b5d8087e0330aac211c85428c8761230ef46a1f2c516d6a2f67fb8803c5",
-                "sha256:2c7c4266c1b61eb429e8aeb7d8ed6a3bfe6c890a1788b18dbec090c35c6b93fa",
-                "sha256:2d68a8e8a3a816629283faf82358d8c93fe5bd974dd2704152394a3de4cec22a",
-                "sha256:344b89384c250ba6a4ce1786e04d01500e4dac0f4137ceebcaad12973c0ac0b3",
-                "sha256:3455ecc46ea443b5f7d9c2f946ce4017745e017b0d0f8b99c92564eff97e97f5",
-                "sha256:3d544a614055b131111bed6edfa1cb0fb082a7265761bcb03321f2dd7b5c6c48",
-                "sha256:3e5c26905aa651cc8c0ddc45e0e5dea2a1296f70bdc96af17aee9d0493280a17",
-                "sha256:3f5cc8c7bc99d2bbcd704cef165ca7d155cd6464c86cbda8339026a42d219397",
-                "sha256:4992266817169997854f81df7f6db7bdcda1609972d8ffd6919252f09ec3c0f6",
-                "sha256:4d55528ef13af4b4e074d067977b1f61408602f53ae4537dccf42ba665c2c7bd",
-                "sha256:576da63eae7809f375932bfcbca2cf20620a1915bf2fedce4b9cc8491eceefe3",
-                "sha256:58fc4d66ee349a23dbf08c7e964120dc9027059566e29cf0ce6205d590ed7eca",
-                "sha256:5b9bf77008f2c55dabbd099fd3ac87009471d223a1c7ebea36873d39511b780a",
-                "sha256:5e7996aed3f65667c6dcc8302a69368435a87c2364079a066750a2eac75ea01e",
-                "sha256:5f7487be65b9c2c510819e744e375bd41b929a97e5915c4852a82fbb085df62c",
-                "sha256:6388e4e95a26717b94a05ced084e19da4d92aca883f392dffcf8e48c8e221a24",
-                "sha256:65af12f70355de29e1092f319f85a3467f4005e959ab65129cb697169ce94b86",
-                "sha256:668d2b45d62c68c7a370ac3dce108ffda482b0a0f50abd8b4c604a813a59e08f",
-                "sha256:71333c22f7cf5f0480b59a0aef21f652cf9bbaa9679ad261b405b65a57511d1e",
-                "sha256:7150b83b3e3ddaac81a8bb6a9b5f93117674a0e7a2b5a5b32ab31fdfea6df27f",
-                "sha256:748e472345c3a82cfb462d0dff998a7bf43e621eed73374cb19f307e97e08a83",
-                "sha256:75dbfd41a61bc1fb0536bf7b1abf272dc115c53d4d77db770cd65d46d4520882",
-                "sha256:7618a082c55cf038eede4a918c1001cc8a4411dfe508dc762659bcd48d8f4c6e",
-                "sha256:780fcb855be29153901c67fc9c5633d48aebef21b90aa72812fa181d731c6b00",
-                "sha256:78d10c431073dc6ebceed35ab22948a016cc2b5120963c13a41e38bdde4a7212",
-                "sha256:7a3a3d3e4f1e3cd2a67b93a0b6ed0f2499e33f47cc568e3a0023e405abdc0ff1",
-                "sha256:7b6975d3763d0952c111700c0634968419268e6bbc0b55fe71138987fa66f309",
-                "sha256:80772e3bda6787510d9620bc0c7572be404a922f8ccdfd436bf6c3778119464c",
-                "sha256:80992eb20755701753e30a6952a96aa58f353d12a65ad3c9d48a8da5ec4690cf",
-                "sha256:841128a22e6ac04070a0f84776d07e9c38c4dcce8e28792a95e45fc621605517",
-                "sha256:861d25ae0985a1dd5297fee35f476b60c6029e2e6e19847d5b4d0a43a390b696",
-                "sha256:872f3dcaa8bf2245944861d7311179d2c0c9b2aaa7d3b464d99a7c2e401f01fa",
-                "sha256:87c93b25d538c433fb053da6228c6290117ba53ff6a537c133b0f2087948a582",
-                "sha256:8856aa76839dc234d3469f1e270918ce6bec1d6a601eba928f45d68a15f04fc3",
-                "sha256:885e023e73ce09b11b89ab91fc60f35d80878d2c19d6213a32b42ff36543c291",
-                "sha256:899b5e7e2d5a8bc92aa533c2d4e55e5ebba095c485568a5e4bedbc163421259a",
-                "sha256:8ce8caa29ebbdcde67e5fd652c811d34bc01f249dbc0d61e5cc4db05ae79a83b",
-                "sha256:8e1c68303ccf7fceb50fbab79064a2636119fd9aca121f28453709283dbca727",
-                "sha256:8e7e2b3577e97fa43c2c2b12a16139b2cedbd0770235d5179c0412b4794efd9b",
-                "sha256:92f05fc7d832e970047662b3440b190d24ea04f8d3c760e33e7163b67308c878",
-                "sha256:97f5811df21703446b42303475b8b855ee07d6ab6cdf8565eff115540624f25d",
-                "sha256:9affee8cb1ec453382c27eb9043378ab32f49cd4bc24a24275f5c39bf186c279",
-                "sha256:a2da4a8c6d465fde36cea7d54bf47b5cf089073452f0e47c8632ecb9dec23c07",
-                "sha256:a6903cdca64f1e301af9be424798328c1fe3b4b14aede35f04510989fc72f012",
-                "sha256:a8ab1adf04ae2d6d65835995218fd3f3eb644fe20655ca8ee233e2c7270ff53b",
-                "sha256:a8edd467551c1102dc0f5754ab55cd0703431cd3044edf8c8e7d9208d63fa453",
-                "sha256:ac00c41dd315d147b129976204839ca9de699d83519ff1272afbe4fb9d362d12",
-                "sha256:ad277f74b1c164f7248afa968700e410651eb858d7c160d109fb451dc45a2f09",
-                "sha256:ae46a50d235f1631d9ec4670503f7b30405103034830bc13df29fd947207f795",
-                "sha256:afe6b5a04b2ab1aa89bad32ca47bf71358e7302a06fdfdad857389dca8fb5f04",
-                "sha256:b1cb078f54af0abd835ca76f93a3152565b73be0f056264da45117d0adf5e99c",
-                "sha256:b25136212a3d064a8f0b9ebbb6c57094c5229e0de76d15c79b76feff26aeb7b8",
-                "sha256:b3226b246facae14909b465061ddcfa2dfeadb6a64f407f24300d42d69bcb1a1",
-                "sha256:b98e75b21fc2ba5285aef8efaf34131d16af1c38df36bdca2f50634bea2d3060",
-                "sha256:bbd7b24d108509a1b9b6679fcc1166a7dd031dbef1f3c2c73788f42e3ebb3beb",
-                "sha256:bed57543c99249ab3a4586ddc8786529fbc33309e5e8a1351802a06ca2baf4c2",
-                "sha256:c0583f69522732bdd79dca4cd3873e63a29acf4a299769c7541f2ca1e4dd4bc6",
-                "sha256:c1e0e9916301e3b3d970814b1439ca59487f0616d30f36a44cead66ee1748c31",
-                "sha256:c651847545422c8131660704c58606d841e228ed576c8f1666d98b3d318f89da",
-                "sha256:c7853f27195598e550fe089f78f0732c66ee1d1f0eaae8ad081589a5a2f5d4af",
-                "sha256:cbae50d352e4717ffc22c566afc2d0da744380e87ed44a144508e3fb9114a3f4",
-                "sha256:cdbed8f21204398f47de39b0a9b180d7e571f02dfb18bf5f1b618e238454b685",
-                "sha256:d08395595c42bcd82c3608762ce734504c6d025eef1c06f42326a6023a584186",
-                "sha256:d4639111e73997567343df6551da9dd90d66aece1b9fc26c786d328439488103",
-                "sha256:d63787f289944cc4bde518ad2b5e70a4f0d6e2ce76324635359c74c113fd188f",
-                "sha256:d6d5f061f6a2aa55790b9e64a23dfd87b6664ab56e24cd06c78eb43986cb260b",
-                "sha256:d7865df1fb564092bcf46dac61b5def25342faf6352e4bc0e61a286e3fa26a3d",
-                "sha256:db6585b600b2e76e98131e0ac0e5195759082b51687ad0c94505970c90718f4a",
-                "sha256:e36d7369363d2707d5f68950a64c4e025991eb0177db01ccb6aa6facae48b69f",
-                "sha256:e7947d9a6264c727a556541b1630296bbd5d0a05068d21c38dde8e7a1c703ef0",
-                "sha256:eb2d59bc196e6d3b1827c7db06c1a898bfa0787c0574af398e65ccf2e97c0fbe",
-                "sha256:ee9c2f6ca9774c2c24bbf7b23086264e6b5fa178201450535ec0859739e6f78d",
-                "sha256:f4760e1b02173f4155203054f77a5dc0b4078de7645c922b208d28e7eb99f3e2",
-                "sha256:f70bec8a14a692be6dbe7ce8aab303e88df891cbd4a39af091f90b6702e28055",
-                "sha256:f869e34d2326e417baee430ae998e91412cc8e7fdd83d979277a90a0e79a5b47",
-                "sha256:f8b9a7cd381970e64849070aca7c32d53ab7d96c66db6c2ef7aa23c6e803f514",
-                "sha256:f99d74ddf9d3b6126b509e81865f89bd1283e3fc1b568b68cd7bd9dfa15583d7",
-                "sha256:f9e7e493ded7042712a374471203dd43ae3fff5b81e3de1a0513fa241af9fd41",
-                "sha256:fc72ae476732cdb7b2c1acb5af23b478b8a0d4b6fcf19b90dd150291e0d5b26b",
-                "sha256:fccbf0cd3411719e4c9426755df90bf3449d9fc5a89f077f4a7f1abd4f70c910",
-                "sha256:ffcf18ad3edf1c170e27e88b10282a2c449aa0358659592462448d71b2000cfc"
+                "sha256:00e97d43a36811b78fa9ad9d3329bf34f76a31e891a7031a2ac01450c9b168ab",
+                "sha256:013d6c784150d10236a74b4094a79d96a256b814457e388fc5a4ba9efe24c402",
+                "sha256:0188b580c490bccb031e9b67e9e8c695a3c44ac5e06218b152361eca847317c3",
+                "sha256:02945ae38fd78efc40900f509890de84cfd5ffe2cd2939eeb3a8800dc68b87cb",
+                "sha256:02b4a2e28eb24dac4ef43dda4f6a6f7766e355179b143f7d0c76a1c5488a307b",
+                "sha256:0527c97dcd8bb983822ee31d3760187083fd3ba18ac4dd22cf5347c89d5628f4",
+                "sha256:05a1382905026bdd560f806c8c7c16e0f3e3fb359ba8868203ca6e5799884968",
+                "sha256:0b309908b6ff5ffbf6394818cb73b5a2a74073acee2c57fe8719046389aeff0d",
+                "sha256:0fc625059b83695fbb4fc8b7a8b66fa94ff9c7b78c84fb9986cd53ff88a28d80",
+                "sha256:177c033e467a66a054dd3a9534167234a3d0b2e41445807b13b626e01da25d92",
+                "sha256:18909093944727e068ebfc92e2e6ed1c4fa44135507c1c0555213ce211c53214",
+                "sha256:1adb04e4b4e41bf30aaa77eeb169c1b9ba9e5010e2e6ce8d6c17e1446edc9b68",
+                "sha256:1ed3d5385d14be894e12a9033be989e012214a9811e7194849c94032ad69682a",
+                "sha256:203eb1532d51591d32e8dfafd60b5d31347ea7278c8da02b4b550287f6abe28b",
+                "sha256:213f9ef5c02ec2f883c1075d25a873149daadbaea50d18d622e9db55ec9849c2",
+                "sha256:2275f1a022e2383da5d2d101fe11ccdcbae799148c4b83260a4b9309fa3e1fc2",
+                "sha256:22e6de18f00583f06928cc8d0993104ecc62f7c6da6478db2255de89a30e45d1",
+                "sha256:289073f68452b96e70990085324be7223944c7409973d13ddfe0eea1c1b5663b",
+                "sha256:29ec8507664f94cc08457d98cfc41c3cdbddfa8952438e644177a29b04937876",
+                "sha256:2a55631b93e47956fbc97d69ba2054a8c6a4016f9a3064ec4e031f5f1030cb90",
+                "sha256:2a86d246a160d98d820ee7d02dc18c923c228de095be362e57b9fd8970b2c4a1",
+                "sha256:2bca97521ee786087f0c5ef318fef3eef0266a9c3deff88205523cf353af7394",
+                "sha256:2c8fc6c841ada60a86d29c9ebe2e8757c47eda6553f3596c560e59ca6e9b6fa1",
+                "sha256:2cd0c9fb5d40887500b4ed818770c68ab4fa6e0395d286f9704be6751b1b7d98",
+                "sha256:2d27d08056fcd61ff47a0cd8407eff4d3e816c82cb6b9c6f0ce9a0ad49225f81",
+                "sha256:2ffbf1b38c88d0466de542e91b08225d51782282512f8e2b11715126c41fda48",
+                "sha256:3fd503c27e7b7034128e30847ecdb4bff4ca5e60f29ad022a9f66ae8940d54ac",
+                "sha256:3ff1f585a0fdc1415bd733b804f33d386064a308672249b14828130dd43e7c31",
+                "sha256:41bd430b7b63aa802c02964e331ac0b177148fef5f807d2c90d05ce71a52b4d4",
+                "sha256:43e9b1531d6a898bdf086acb75c41265c7ec4331267d7619148d407efc72bd24",
+                "sha256:46af4a742b90c7460e94214f923452c2c1d050a9da1d2b8d4c70cbc045e692b7",
+                "sha256:46c4c550bf59ce05d6bff2c98053822549aaf9fbaf81103edea325e03350bca1",
+                "sha256:4969592e3cdeefa4cbb15a26cec102cbd4a1d6e5b695fac9fa026e19741138c8",
+                "sha256:4a0536ed2b9297c75104e1a3da330828ba1b2639fa53b38d396f98bf7e3c68df",
+                "sha256:4a96147791e49e84207dd1530109aa0e9eeaf1c8b7a59f150047fc0fcdf9bb64",
+                "sha256:4c7f9d70f99e1fbcbf57c75328b80e1c0a7f6cad43e75efa90a97221be5efe15",
+                "sha256:4e8474f7233fe1949ce4e03bea698a600c2d5d6b51dab6d6e6336dbe69acf23e",
+                "sha256:4f1b804cfad04f862d6a84af9d1ad941b06f671878f0f7ecad6c92007d423de6",
+                "sha256:529aab727f54a937085184e7436e1d0e19975cf10115eda12d37a683e4ee5342",
+                "sha256:5612b0b1de8d5114520094bd5fc3d04eb8af6f3e10d48ef05b7c8e77c1fd9545",
+                "sha256:56777c57246e048908b550af9b81b0ec9cf804fd47cb7502ccd93238bd6025c2",
+                "sha256:56ba7c1100ed079527f2b995bf5486a2e557e6d5b733c52e8947476338815b69",
+                "sha256:59d222086daa55421d599609b32d0ebe544e57654c4a0a1490c54a7ebaa67561",
+                "sha256:5aba767e64b494483ad60c4873bec78d16205a21f8247c99749bd990d9c846c2",
+                "sha256:5d5eaf988951f6ecb6854ca3300b87123599c711183c83da7ce39717a7cbdbce",
+                "sha256:73da69e1f612c3e682e34dcb971272d90d6f27b2c99acff444ca455a89978574",
+                "sha256:75c8766734ac0053e1d683567e65e85306c4ec62631b0591caeb287ac8f72e08",
+                "sha256:75eea40355a8690459c7291ce6c8ce39c27bd223675c7da6619f510c728feb97",
+                "sha256:80c3cf46511653f94dfe07c7c79ab105c4164d6e1dfcb35b7214fb9af53eaef4",
+                "sha256:8557c807388e6617161fe51b1a4747ea8d1133f2d2ad8e79583439abebe58fbd",
+                "sha256:89438e8885a186c69fe31f7ef98bb2bf29688c466c3caf9060f404c0be89ae80",
+                "sha256:899b03a3be785a7e1ff84b237da71f0efa2f021512f147dd34ffdf7aa82cb678",
+                "sha256:8de9b88f0cbac73cfed34220d13c57849e62a7099a714b929142425e926d223a",
+                "sha256:8f4d561f4728f825e3b793a53064b606ca0b6fc264f67d09e54af452aafc5b82",
+                "sha256:907b214da5d2fcff0b6ddb83de1333890ca92abaf4bbf8d9c61dc1b95c87fd6e",
+                "sha256:9118de88c16947eaf5b92f749e65b0501ea69e7c2be7bd6aefc12551622360e1",
+                "sha256:9568764e72d85cf7855ca78b48e07ed1be47bf230e2cea8dabda3c95f660b0ff",
+                "sha256:9c74cbee9e532dc34371127f7686d6953e5153a1f22beab7f953d95ee4a0fe09",
+                "sha256:9cdfd649011ce2d90cb0dd304c5aba1190fac0c266d19a9e2b96b81cfd150a09",
+                "sha256:9f00d54b18dd837f1431d66b076737deb7c29ce3ebb8412ceaf44d5e1954ac0c",
+                "sha256:a2f416cdfe92f5fbb77177f5f3f7830059d1582db05f2c7119bf80069d1ab69b",
+                "sha256:a4cb372e22e9c879bd9a9cc9b20b7c1fbf30a605ac953da45ecec05d8a6e1c77",
+                "sha256:a65de5c02884760a14a58304fb6303f9ddfc582e630f385daea871e1bdb18686",
+                "sha256:aa3b3a43dabc4cc57a7800f526cbe03f71c69121e21b863fdf497b59b462b163",
+                "sha256:ab0f7aabdbce4a202e013083eeab71afdb85efa405dc4a06fea98cde81204675",
+                "sha256:abe081453166e206e3a8c6d8ace57214c17b6d9477d7601ac14a365344dbc1f4",
+                "sha256:ae141c9017f8f473a6ee07a9425da021816a9f8c0683c2e5442f0ccf56b0fc62",
+                "sha256:af52078719209bef33e38131486fd784832dd8d1dc9b85f00a44f6e7437dd021",
+                "sha256:b00150a9a3fd0a8efaa90bc2696c105b04039d50763dd1c95a34c88c5966cb57",
+                "sha256:b2660000e1a113869c86eb5cc07f3343467490f3cd9d0299f81da9ddae7137b7",
+                "sha256:b3eb1a0d2b6d232d1bcdfc3fcc5f7b004ab3fbd9203011a3172f051d4527c0b6",
+                "sha256:b589d93a60e78fe55d5bc76ee8c2bf945dbdbb7cd16044c53e0307604e448de1",
+                "sha256:b8578fc6c8bdd0201327503720fa581000b4bd3934abbf07e2628d1ad3de157d",
+                "sha256:ba1b28e44f611f3f2b436bd8290050a61db4b59a8e24be4465f44897936b3824",
+                "sha256:bb44644371eaa29a3aba7b69b1862d0d56f073bb7585baa32e4271a71a91ee82",
+                "sha256:bcde80aefe7054fad6277762fb7e9d35c72ea479a485ae1bb14629c640987b30",
+                "sha256:bd1142d22fdb183a0fff66d79134bf644401437fed874f81066d314c67ee193c",
+                "sha256:bf77f9017fcfa1232f98598a637406e6c33982ccba8a5922339575c3e2b90ea5",
+                "sha256:c2772bb95062e3f9774140205cd65d8997e39620715486cf5f843cf4ad8f744c",
+                "sha256:c4ecc4e9a5d73a816cae36ee6b5d8b7a0c72013cae1e101406e832887c3dc2d8",
+                "sha256:c86231c66e4f422e7c13ea6200bb4048b3016c8bfd11b4fd0dabd04d2c8e3501",
+                "sha256:c8f6526df47953b07c45b95c4d1da6b9a0861c0e5da0271db96bb1d807825412",
+                "sha256:ccfb77f6dc8abffa6f1c7e3975ed9070a41ce5fcc11154d2bead8c1baa940f09",
+                "sha256:d9d7efaad48b859053b90dedd69bc92f2095084251e732e4c57ac9726bcb1e64",
+                "sha256:dd91a7d7a9ce7f4983097c91ce211f3e5569cc21caa16f2692298a07e396f82b",
+                "sha256:de4a2fd524993578fe093044f291b4b24aab134390030b3b9b5f87fd41ab7e75",
+                "sha256:df61f818edf7c8626bfa392f825860fb670b5f8336e238eb0ec7e2a5689cdded",
+                "sha256:e1147bc3d0dd1e549d991110d0a09557ec9f925dbc1ca62871fcdab2ec9d716b",
+                "sha256:e1954f4b239d1a92081647eecfd51cbfd08ea16eb743b8af1cd0113258feea14",
+                "sha256:e281b71922208e00886e4b7ffbfcf27874486364f177418ab676f102130e7ec9",
+                "sha256:e69737bd56006a86fd5a78b2b85447580a6138c930a75eb9ef39fe03d90782b1",
+                "sha256:e82b4a70cc67094f3f3fd77579702f48fcf1de7bdc67d79b8f1e24d089a6162c",
+                "sha256:e92e5817eb6bfed23aa5e45bfe30647b83602bdd6f9e25d63524d4e6258458b0",
+                "sha256:eaba0613c759ebf95988a84f766ca6b7432d55ce399194f95dde588ad1be0878",
+                "sha256:edd74b760a6bb950397e7a7bd2f38e6700f6525062650b1d77c6d851b82f02c2",
+                "sha256:f40abbcc0a7d9a8a80870af839d317e6932533f98682aabd977add6c53beeb23",
+                "sha256:fce7a8ee8d0f682c953c0188735d823f0fcb62779bf92cd6ba473a8e730e26ad"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.10.0"
+            "version": "==0.10.2"
         },
         "s3transfer": {
             "hashes": [
@@ -2014,11 +2048,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:3d4dfa6d95f1b101d695a6160a7626e15583af71a5f52176efa5d39a054d475d",
-                "sha256:3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b"
+                "sha256:00478ca80aeebeecb2f288d3206b0de568df5cd2b8fada1209843cc9a8d88a48",
+                "sha256:af3d5949030c3f493f550876b2fd1dd5ec66689c4ee5d5344f009746f71fd5a8"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==68.1.2"
+            "version": "==68.2.0"
         },
         "simplejson": {
             "hashes": [
@@ -2108,7 +2142,7 @@
                 "sha256:f41915a4e1f059dfad614b187bc06021fefb5fc5255bfe63abf8247d2f7a646a",
                 "sha256:f96def94576f857abf58e031ce881b5a3fc25cbec64b2bc4824824a8a4367af9"
             ],
-            "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2'",
+            "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==3.19.1"
         },
         "six": {
@@ -2116,7 +2150,7 @@
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
                 "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.16.0"
         },
         "smmap": {
@@ -2137,11 +2171,11 @@
         },
         "soupsieve": {
             "hashes": [
-                "sha256:1c1bfee6819544a3447586c889157365a27e10d88cde3ad3da0cf0ddf646feb8",
-                "sha256:89d12b2d5dfcd2c9e8c22326da9d9aa9cb3dfab0a83a024f05704076ee8d35ea"
+                "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690",
+                "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==2.4.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.5"
         },
         "sqlalchemy": {
             "hashes": [
@@ -2820,6 +2854,80 @@
             "index": "pypi",
             "version": "==5.12.0"
         },
+        "jinja2": {
+            "hashes": [
+                "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852",
+                "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"
+            ],
+            "index": "pypi",
+            "version": "==3.1.2"
+        },
+        "markupsafe": {
+            "hashes": [
+                "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e",
+                "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e",
+                "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431",
+                "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686",
+                "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c",
+                "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559",
+                "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc",
+                "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb",
+                "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939",
+                "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c",
+                "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0",
+                "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4",
+                "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9",
+                "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575",
+                "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba",
+                "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d",
+                "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd",
+                "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3",
+                "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00",
+                "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155",
+                "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac",
+                "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52",
+                "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f",
+                "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8",
+                "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b",
+                "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007",
+                "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24",
+                "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea",
+                "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198",
+                "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0",
+                "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee",
+                "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be",
+                "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2",
+                "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1",
+                "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707",
+                "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6",
+                "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c",
+                "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58",
+                "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823",
+                "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779",
+                "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636",
+                "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c",
+                "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad",
+                "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee",
+                "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc",
+                "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2",
+                "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48",
+                "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7",
+                "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e",
+                "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b",
+                "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa",
+                "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5",
+                "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e",
+                "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb",
+                "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9",
+                "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57",
+                "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc",
+                "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc",
+                "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2",
+                "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.1.3"
+        },
         "mccabe": {
             "hashes": [
                 "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325",
@@ -2900,14 +3008,6 @@
             "markers": "python_version >= '3.8'",
             "version": "==1.3.0"
         },
-        "py": {
-            "hashes": [
-                "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719",
-                "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==1.11.0"
-        },
         "pycodestyle": {
             "hashes": [
                 "sha256:259bcc17857d8a8b3b4a2327324b79e5f020a13c16074670f9c8c8f872ea76d0",
@@ -2949,11 +3049,11 @@
         },
         "pytest-html": {
             "hashes": [
-                "sha256:868c08564a68d8b2c26866f1e33178419bb35b1e127c33784a28622eb827f3f3",
-                "sha256:c4e2f4bb0bffc437f51ad2174a8a3e71df81bbc2f6894604e604af18fbe687c3"
+                "sha256:3b473cc278272f8b5a34cd3bf10f88ac5fcb17cb5af22f9323514af00c310e64",
+                "sha256:79c4677ed6196417bf290d8b81f706342ae49f726f623728efa3f7dfff09f8eb"
             ],
             "index": "pypi",
-            "version": "==3.2.0"
+            "version": "==4.0.0"
         },
         "pytest-metadata": {
             "hashes": [
diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index d341836f5f..32c1a853b4 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -100,8 +100,8 @@ def save_checkpoint(model: torch.nn.Module, path: str, cfg: Any):
     if cfg.environment.use_deepspeed:
         if path is not None:
             # gather model params from all ranks
-            if hasattr(cfg.training, "lora") and cfg.training.lora:
-                model.backbone.save_pretrained(path)
+            # if hasattr(cfg.training, "lora") and cfg.training.lora:
+            #     model.backbone.save_pretrained(path)
             model.save_checkpoint(os.path.join(path, "ds_checkpoint"))
             if cfg.environment._local_rank == 0:
                 # load to cpu
@@ -115,8 +115,8 @@ def save_checkpoint(model: torch.nn.Module, path: str, cfg: Any):
     else:
         if cfg.environment._local_rank == 0:
             model = unwrap_model(model)
-            if hasattr(cfg.training, "lora") and cfg.training.lora:
-                model.backbone.save_pretrained(path)
+            # if hasattr(cfg.training, "lora") and cfg.training.lora:
+            #     model.backbone.save_pretrained(path)
             checkpoint = {"model": model.state_dict()}
             if path is not None:
                 torch.save(checkpoint, os.path.join(path, "checkpoint.pth"))
diff --git a/train.py b/train.py
index d78c8a6a46..dd53b3c174 100644
--- a/train.py
+++ b/train.py
@@ -918,15 +918,16 @@ def run(cfg: Any) -> None:
 
     experiment_path = f"{cfg.output_directory}"
 
+    if cfg.training.epochs == 0:
+        checkpoint_path = cfg.output_directory
+        logger.info(
+            f"Saving last model checkpoint: "
+            f"val_loss {val_loss:.5}, val_{cfg.prediction.metric} "
+            f"{val_metric:.5} to {checkpoint_path}"
+        )
+        save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
+
     if cfg.environment._local_rank == 0:
-        if cfg.training.epochs == 0:
-            checkpoint_path = cfg.output_directory
-            logger.info(
-                f"Saving last model checkpoint: "
-                f"val_loss {val_loss:.5}, val_{cfg.prediction.metric} "
-                f"{val_metric:.5} to {checkpoint_path}"
-            )
-            save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
 
         save_config_yaml(f"{cfg.output_directory}/cfg.yaml", cfg)
 

From bd1e134c916a98526576da297da45140ce2efbc8 Mon Sep 17 00:00:00 2001
From: Philipp Singer <killver@gmail.com>
Date: Fri, 8 Sep 2023 14:30:52 +0000
Subject: [PATCH 38/52] lock

---
 Pipfile.lock | 51 ++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 50 insertions(+), 1 deletion(-)

diff --git a/Pipfile.lock b/Pipfile.lock
index 8ff36235da..7b3d4a059f 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "20732924c37ba84a1dd1804d15edc2e3681c7168e1709738c906bca9618177cb"
+            "sha256": "3c74668c1f0f6d9d48bab21432ba08d5612ed2e53a8dfb69fc69346f82588072"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -609,6 +609,13 @@
             "index": "pypi",
             "version": "==2.14.5"
         },
+        "deepspeed": {
+            "hashes": [
+                "sha256:286bb1f01ea7340c9e18d78a562393dd1c12cc1406f1f5996f4be1d6984360ef"
+            ],
+            "index": "pypi",
+            "version": "==0.10.2"
+        },
         "dill": {
             "hashes": [
                 "sha256:76b122c08ef4ce2eedcd4d1abd8e641114bfc6c2867f49f3c41facf65bf19f5e",
@@ -1605,6 +1612,48 @@
             ],
             "version": "==2.21"
         },
+        "pydantic": {
+            "hashes": [
+                "sha256:0fe8a415cea8f340e7a9af9c54fc71a649b43e8ca3cc732986116b3cb135d303",
+                "sha256:1289c180abd4bd4555bb927c42ee42abc3aee02b0fb2d1223fb7c6e5bef87dbe",
+                "sha256:1eb2085c13bce1612da8537b2d90f549c8cbb05c67e8f22854e201bde5d98a47",
+                "sha256:2031de0967c279df0d8a1c72b4ffc411ecd06bac607a212892757db7462fc494",
+                "sha256:2a7bac939fa326db1ab741c9d7f44c565a1d1e80908b3797f7f81a4f86bc8d33",
+                "sha256:2d5a58feb9a39f481eda4d5ca220aa8b9d4f21a41274760b9bc66bfd72595b86",
+                "sha256:2f9a6fab5f82ada41d56b0602606a5506aab165ca54e52bc4545028382ef1c5d",
+                "sha256:2fcfb5296d7877af406ba1547dfde9943b1256d8928732267e2653c26938cd9c",
+                "sha256:549a8e3d81df0a85226963611950b12d2d334f214436a19537b2efed61b7639a",
+                "sha256:598da88dfa127b666852bef6d0d796573a8cf5009ffd62104094a4fe39599565",
+                "sha256:5d1197e462e0364906cbc19681605cb7c036f2475c899b6f296104ad42b9f5fb",
+                "sha256:69328e15cfda2c392da4e713443c7dbffa1505bc9d566e71e55abe14c97ddc62",
+                "sha256:6a9dfa722316f4acf4460afdf5d41d5246a80e249c7ff475c43a3a1e9d75cf62",
+                "sha256:6b30bcb8cbfccfcf02acb8f1a261143fab622831d9c0989707e0e659f77a18e0",
+                "sha256:6c076be61cd0177a8433c0adcb03475baf4ee91edf5a4e550161ad57fc90f523",
+                "sha256:771735dc43cf8383959dc9b90aa281f0b6092321ca98677c5fb6125a6f56d58d",
+                "sha256:795e34e6cc065f8f498c89b894a3c6da294a936ee71e644e4bd44de048af1405",
+                "sha256:87afda5539d5140cb8ba9e8b8c8865cb5b1463924d38490d73d3ccfd80896b3f",
+                "sha256:8fb2aa3ab3728d950bcc885a2e9eff6c8fc40bc0b7bb434e555c215491bcf48b",
+                "sha256:a1fcb59f2f355ec350073af41d927bf83a63b50e640f4dbaa01053a28b7a7718",
+                "sha256:a5e7add47a5b5a40c49b3036d464e3c7802f8ae0d1e66035ea16aa5b7a3923ed",
+                "sha256:a73f489aebd0c2121ed974054cb2759af8a9f747de120acd2c3394cf84176ccb",
+                "sha256:ab26038b8375581dc832a63c948f261ae0aa21f1d34c1293469f135fa92972a5",
+                "sha256:b0d191db0f92dfcb1dec210ca244fdae5cbe918c6050b342d619c09d31eea0cc",
+                "sha256:b749a43aa51e32839c9d71dc67eb1e4221bb04af1033a32e3923d46f9effa942",
+                "sha256:b7ccf02d7eb340b216ec33e53a3a629856afe1c6e0ef91d84a4e6f2fb2ca70fe",
+                "sha256:ba5b2e6fe6ca2b7e013398bc7d7b170e21cce322d266ffcd57cca313e54fb246",
+                "sha256:ba5c4a8552bff16c61882db58544116d021d0b31ee7c66958d14cf386a5b5350",
+                "sha256:c79e6a11a07da7374f46970410b41d5e266f7f38f6a17a9c4823db80dadf4303",
+                "sha256:ca48477862372ac3770969b9d75f1bf66131d386dba79506c46d75e6b48c1e09",
+                "sha256:dea7adcc33d5d105896401a1f37d56b47d443a2b2605ff8a969a0ed5543f7e33",
+                "sha256:e0a16d274b588767602b7646fa05af2782576a6cf1022f4ba74cbb4db66f6ca8",
+                "sha256:e4129b528c6baa99a429f97ce733fff478ec955513630e61b49804b6cf9b224a",
+                "sha256:e5f805d2d5d0a41633651a73fa4ecdd0b3d7a49de4ec3fadf062fe16501ddbf1",
+                "sha256:ef6c96b2baa2100ec91a4b428f80d8f28a3c9e53568219b6c298c1125572ebc6",
+                "sha256:fdbdd1d630195689f325c9ef1a12900524dceb503b00a987663ff4f58669b93d"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.10.12"
+        },
         "pyjwt": {
             "hashes": [
                 "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de",

From 6f81182e313755acd28f72b16293b205be59b8e5 Mon Sep 17 00:00:00 2001
From: psinger <psinger@users.noreply.github.com>
Date: Fri, 8 Sep 2023 14:35:12 +0000
Subject: [PATCH 39/52] Update requirements.txt

---
 requirements.txt | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 0b94714647..29d1ef6b86 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -30,6 +30,7 @@ coolname==2.2.0
 cramjam==2.7.0 ; python_version >= '3.7'
 cryptography==41.0.3 ; python_version >= '3.7'
 datasets==2.14.5
+deepspeed==0.10.2
 dill==0.3.7
 docopt==0.6.2
 einops==0.6.1
@@ -43,7 +44,7 @@ future==0.18.3 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.
 gitdb==4.0.10 ; python_version >= '3.7'
 gitpython==3.1.35 ; python_version >= '3.7'
 gputil==1.4.0
-greenlet==2.0.2 ; python_version >= '3' and (platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32'))))))
+greenlet==2.0.2 ; python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))
 h11==0.14.0 ; python_version >= '3.7'
 h2o-wave==0.26.2
 hf-transfer==0.1.3
@@ -87,8 +88,9 @@ psutil==5.9.5 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2
 py-cpuinfo==9.0.0
 pyarrow==8.0.0
 pycparser==2.21
+pydantic==1.10.12 ; python_version >= '3.7'
 pyjwt==2.8.0 ; python_version >= '3.7'
-python-dateutil==2.8.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
+python-dateutil==2.8.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 python-editor==1.0.4
 python-slugify==8.0.1 ; python_version >= '3.7'
 pytz==2023.3.post1

From 62fc9c5052076c455b4fa3affe027c453bd0d8c5 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Tue, 26 Sep 2023 01:54:24 -0700
Subject: [PATCH 40/52] improve model saving for deepspeed

---
 llm_studio/src/utils/modeling_utils.py | 35 +++++++++++++++-----------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index 879ebb3134..a19f6302c0 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -98,24 +98,26 @@ def save_checkpoint(model: torch.nn.Module, path: str, cfg: Any):
 
     if cfg.environment.use_deepspeed:
         if path is not None:
-            # gather model params from all ranks
-            # if hasattr(cfg.training, "lora") and cfg.training.lora:
-            #     model.backbone.save_pretrained(path)
-            model.save_checkpoint(os.path.join(path, "ds_checkpoint"))
-            if cfg.environment._local_rank == 0:
-                # load to cpu
-                state_dict = get_fp32_state_dict_from_zero_checkpoint(
-                    os.path.join(path, "ds_checkpoint")
+            # gather model params from all ranks when using Deepspeed
+            if not model.save_16bit_model(path, "checkpoint.pth"):
+                logger.warning(
+                    "deepspeed.save_16bit_model didn't save the model, since"
+                    " stage3_gather_16bit_weights_on_model_save=False."
+                    " Saving the full checkpoint instead"
                 )
-                # save as normal checkpoint that can be loaded by `load_state_dict`
-                checkpoint = {"model": state_dict}
-                torch.save(checkpoint, os.path.join(path, "checkpoint.pth"))
-                shutil.rmtree(os.path.join(path, "ds_checkpoint"))
+                model.save_checkpoint(os.path.join(path, "ds_checkpoint"))
+                if cfg.environment._local_rank == 0:
+                    # load to cpu
+                    state_dict = get_fp32_state_dict_from_zero_checkpoint(
+                        os.path.join(path, "ds_checkpoint")
+                    )
+                    # save as normal checkpoint that can be loaded by `load_state_dict`
+                    checkpoint = {"model": state_dict}
+                    torch.save(checkpoint, os.path.join(path, "checkpoint.pth"))
+                    shutil.rmtree(os.path.join(path, "ds_checkpoint"))
     else:
         if cfg.environment._local_rank == 0:
             model = unwrap_model(model)
-            # if hasattr(cfg.training, "lora") and cfg.training.lora:
-            #     model.backbone.save_pretrained(path)
             checkpoint = {"model": model.state_dict()}
             if path is not None:
                 torch.save(checkpoint, os.path.join(path, "checkpoint.pth"))
@@ -193,7 +195,9 @@ def load_checkpoint(
     if weights_path is None:
         weights_path = cfg.architecture.pretrained_weights
 
-    model_weights = torch.load(weights_path, map_location="cpu")["model"]
+    model_weights = torch.load(weights_path, map_location="cpu")
+    if "model" in model_weights.keys():
+        model_weights = model_weights["model"]
 
     model = load_model_weights(model, model_weights, strict, cfg)
 
@@ -224,6 +228,7 @@ def get_ds_config(cfg: Any):
             # zero3
             "stage3_prefetch_bucket_size": cfg.environment.deepspeed_stage3_prefetch_bucket_size,  # noqa: E501
             "stage3_param_persistence_threshold": cfg.environment.deepspeed_stage3_param_persistence_threshold,  # noqa: E501
+            "stage3_gather_16bit_weights_on_model_save": True,
             # zero3 offload cpu
             # "stage3_max_live_parameters": cfg.environment.deepspeed_stage3_max_live_parameters,  # noqa: E501
             # "stage3_max_reuse_distance": cfg.environment.deepspeed_stage3_max_reuse_distance,  # noqa: E501

From dbbbcdf4ef6c0593a8be9557836ad3b8b7f3ec35 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Tue, 26 Sep 2023 01:58:18 -0700
Subject: [PATCH 41/52] solved INFLIGHT problem

---
 llm_studio/src/utils/modeling_utils.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index a19f6302c0..9af08b0fbd 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -518,12 +518,23 @@ def run_inference(
 
         batch = cfg.dataset.dataset_class.batch_to_device(data, cfg.environment._device)
 
-        with autocast(enabled=cfg.environment.mixed_precision):
-            output = model.forward(batch)
+        if cfg.environment.use_deepspeed:
             if cfg.prediction.metric != "Perplexity":
+                output = {}
                 output["predicted_answer_ids"] = (
-                    unwrap_model(model).generate(batch, cfg).detach().cpu()
+                    model.module.generate(batch, cfg).detach().cpu()
                 )
+            else:
+                output = model.forward(batch)
+        else:
+            with autocast(enabled=cfg.environment.mixed_precision):
+                if cfg.prediction.metric != "Perplexity":
+                    output = {}
+                    output["predicted_answer_ids"] = (
+                        unwrap_model(model).generate(batch, cfg).detach().cpu()
+                    )
+                else:
+                    output = model.forward(batch)
         if contains_nan(output) and cfg.environment.mixed_precision:
             raise LLMModelException(
                 "NaN caught during mixed precision inference. "

From c023d19220a0d1c46126b0aea7ba647be9c87c23 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Tue, 26 Sep 2023 05:10:08 -0700
Subject: [PATCH 42/52] update doc

---
 .../docs/tooltips/experiments/_deepspeed-offload-optimizer.mdx  | 2 +-
 .../docs/tooltips/experiments/_deepspeed-reduce-bucket-size.mdx | 2 +-
 .../experiments/_deepspeed-stage3-max-live-parameters.mdx       | 2 +-
 .../experiments/_deepspeed-stage3-max-reuse-distance.mdx        | 2 +-
 .../_deepspeed-stage3-param-persistence-threshold.mdx           | 2 +-
 .../experiments/_deepspeed-stage3-prefetch-bucket-size.mdx      | 2 +-
 documentation/docs/tooltips/experiments/_use-deepspeed.mdx      | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/documentation/docs/tooltips/experiments/_deepspeed-offload-optimizer.mdx b/documentation/docs/tooltips/experiments/_deepspeed-offload-optimizer.mdx
index ffd3b28c50..d83fa269a0 100644
--- a/documentation/docs/tooltips/experiments/_deepspeed-offload-optimizer.mdx
+++ b/documentation/docs/tooltips/experiments/_deepspeed-offload-optimizer.mdx
@@ -1 +1 @@
-Whether to offload optimizer to cpu for saving more GPU ram during training. Note that turn on offload_optimizer would further make training speed slower.
\ No newline at end of file
+Whether to offload optimizer to cpu for saving more GPU ram during training. Note that turn on offload_optimizer would further slow down training.
\ No newline at end of file
diff --git a/documentation/docs/tooltips/experiments/_deepspeed-reduce-bucket-size.mdx b/documentation/docs/tooltips/experiments/_deepspeed-reduce-bucket-size.mdx
index c6155bcb3d..d2b6d37eb1 100644
--- a/documentation/docs/tooltips/experiments/_deepspeed-reduce-bucket-size.mdx
+++ b/documentation/docs/tooltips/experiments/_deepspeed-reduce-bucket-size.mdx
@@ -1 +1 @@
-Number of elements reduced/allreduced at a time. Limits the memory required for the allgather for large model sizes. Smaller values use less memory, but make training speed slower.
\ No newline at end of file
+Number of elements reduced/allreduced at a time. Limits the memory required for the allgather for large model sizes. Smaller values use less memory, but slow down training.
\ No newline at end of file
diff --git a/documentation/docs/tooltips/experiments/_deepspeed-stage3-max-live-parameters.mdx b/documentation/docs/tooltips/experiments/_deepspeed-stage3-max-live-parameters.mdx
index 3c2bafea9c..7081411219 100644
--- a/documentation/docs/tooltips/experiments/_deepspeed-stage3-max-live-parameters.mdx
+++ b/documentation/docs/tooltips/experiments/_deepspeed-stage3-max-live-parameters.mdx
@@ -1 +1 @@
-The maximum number of parameters resident per GPU before releasing. Smaller values use less memory, but make training speed slower.
\ No newline at end of file
+The maximum number of parameters resident per GPU before releasing. Smaller values use less memory, but slow down training.
\ No newline at end of file
diff --git a/documentation/docs/tooltips/experiments/_deepspeed-stage3-max-reuse-distance.mdx b/documentation/docs/tooltips/experiments/_deepspeed-stage3-max-reuse-distance.mdx
index 268b542995..0e5beb51a6 100644
--- a/documentation/docs/tooltips/experiments/_deepspeed-stage3-max-reuse-distance.mdx
+++ b/documentation/docs/tooltips/experiments/_deepspeed-stage3-max-reuse-distance.mdx
@@ -1 +1 @@
-Do not release a parameter if it will be reused within this threshold of parameters. Smaller values use less memory, but make training speed slower.
\ No newline at end of file
+Do not release a parameter if it will be reused within this threshold of parameters. Smaller values use less memory, but slow down training.
\ No newline at end of file
diff --git a/documentation/docs/tooltips/experiments/_deepspeed-stage3-param-persistence-threshold.mdx b/documentation/docs/tooltips/experiments/_deepspeed-stage3-param-persistence-threshold.mdx
index 7763fafdb6..ab7d461215 100644
--- a/documentation/docs/tooltips/experiments/_deepspeed-stage3-param-persistence-threshold.mdx
+++ b/documentation/docs/tooltips/experiments/_deepspeed-stage3-param-persistence-threshold.mdx
@@ -1 +1 @@
-Do not partition parameters smaller than this threshold. Smaller values use less memory, but can greatly increase communication and make training speed slower. (especially latency-bound messages).
+Do not partition parameters smaller than this threshold. Smaller values use less memory, but can greatly increase communication and slow down training. (especially latency-bound messages).
diff --git a/documentation/docs/tooltips/experiments/_deepspeed-stage3-prefetch-bucket-size.mdx b/documentation/docs/tooltips/experiments/_deepspeed-stage3-prefetch-bucket-size.mdx
index 29cb1f4f26..eb24ae37d7 100644
--- a/documentation/docs/tooltips/experiments/_deepspeed-stage3-prefetch-bucket-size.mdx
+++ b/documentation/docs/tooltips/experiments/_deepspeed-stage3-prefetch-bucket-size.mdx
@@ -1 +1 @@
-Maximum number of parameter elements to fetch ahead of use. Smaller values use less memory, but make training speed slower.
\ No newline at end of file
+Maximum number of parameter elements to fetch ahead of use. Smaller values use less memory, but slow down training..
\ No newline at end of file
diff --git a/documentation/docs/tooltips/experiments/_use-deepspeed.mdx b/documentation/docs/tooltips/experiments/_use-deepspeed.mdx
index 884e2ba04e..6865a6fd03 100644
--- a/documentation/docs/tooltips/experiments/_use-deepspeed.mdx
+++ b/documentation/docs/tooltips/experiments/_use-deepspeed.mdx
@@ -1 +1 @@
-Whether to use deepspeed for saving GPU ram during training. Note that turn on deepspeed would make training speed slower.
\ No newline at end of file
+Whether to use deepspeed for saving GPU ram during training. Note that turning on DeepSpeed can slow down training.
\ No newline at end of file

From 2785f9f2c4a83bec98e90be5389b21fb8a62e04e Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Thu, 28 Sep 2023 05:48:52 -0700
Subject: [PATCH 43/52] deepspeed default push to hub by cpu

---
 llm_studio/app_utils/sections/experiment.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/llm_studio/app_utils/sections/experiment.py b/llm_studio/app_utils/sections/experiment.py
index 7d8fdcf660..4b8e2a1b95 100644
--- a/llm_studio/app_utils/sections/experiment.py
+++ b/llm_studio/app_utils/sections/experiment.py
@@ -1680,7 +1680,9 @@ async def experiment_push_to_huggingface_dialog(q: Q, error: str = ""):
         num_running_queued = len(
             experiments[experiments["status"].isin(["queued", "running"])]
         )
-        if num_running_queued > 0:
+        experiment_path = q.client["experiment/display/experiment_path"]
+        cfg = load_config_yaml(os.path.join(experiment_path, "cfg.yaml"))
+        if num_running_queued > 0 or cfg.environment.use_deepspeed:
             default_device = "cpu"
 
         try:

From aa17c0b1625146fe5579b40946bb0dde4b32a1c2 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Thu, 5 Oct 2023 03:45:04 -0700
Subject: [PATCH 44/52] Revert "improve model saving for deepspeed"

This reverts commit 62fc9c5052076c455b4fa3affe027c453bd0d8c5.
---
 llm_studio/src/utils/modeling_utils.py | 35 +++++++++++---------------
 1 file changed, 15 insertions(+), 20 deletions(-)

diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index 9af08b0fbd..7ea3b842bb 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -98,26 +98,24 @@ def save_checkpoint(model: torch.nn.Module, path: str, cfg: Any):
 
     if cfg.environment.use_deepspeed:
         if path is not None:
-            # gather model params from all ranks when using Deepspeed
-            if not model.save_16bit_model(path, "checkpoint.pth"):
-                logger.warning(
-                    "deepspeed.save_16bit_model didn't save the model, since"
-                    " stage3_gather_16bit_weights_on_model_save=False."
-                    " Saving the full checkpoint instead"
+            # gather model params from all ranks
+            # if hasattr(cfg.training, "lora") and cfg.training.lora:
+            #     model.backbone.save_pretrained(path)
+            model.save_checkpoint(os.path.join(path, "ds_checkpoint"))
+            if cfg.environment._local_rank == 0:
+                # load to cpu
+                state_dict = get_fp32_state_dict_from_zero_checkpoint(
+                    os.path.join(path, "ds_checkpoint")
                 )
-                model.save_checkpoint(os.path.join(path, "ds_checkpoint"))
-                if cfg.environment._local_rank == 0:
-                    # load to cpu
-                    state_dict = get_fp32_state_dict_from_zero_checkpoint(
-                        os.path.join(path, "ds_checkpoint")
-                    )
-                    # save as normal checkpoint that can be loaded by `load_state_dict`
-                    checkpoint = {"model": state_dict}
-                    torch.save(checkpoint, os.path.join(path, "checkpoint.pth"))
-                    shutil.rmtree(os.path.join(path, "ds_checkpoint"))
+                # save as normal checkpoint that can be loaded by `load_state_dict`
+                checkpoint = {"model": state_dict}
+                torch.save(checkpoint, os.path.join(path, "checkpoint.pth"))
+                shutil.rmtree(os.path.join(path, "ds_checkpoint"))
     else:
         if cfg.environment._local_rank == 0:
             model = unwrap_model(model)
+            # if hasattr(cfg.training, "lora") and cfg.training.lora:
+            #     model.backbone.save_pretrained(path)
             checkpoint = {"model": model.state_dict()}
             if path is not None:
                 torch.save(checkpoint, os.path.join(path, "checkpoint.pth"))
@@ -195,9 +193,7 @@ def load_checkpoint(
     if weights_path is None:
         weights_path = cfg.architecture.pretrained_weights
 
-    model_weights = torch.load(weights_path, map_location="cpu")
-    if "model" in model_weights.keys():
-        model_weights = model_weights["model"]
+    model_weights = torch.load(weights_path, map_location="cpu")["model"]
 
     model = load_model_weights(model, model_weights, strict, cfg)
 
@@ -228,7 +224,6 @@ def get_ds_config(cfg: Any):
             # zero3
             "stage3_prefetch_bucket_size": cfg.environment.deepspeed_stage3_prefetch_bucket_size,  # noqa: E501
             "stage3_param_persistence_threshold": cfg.environment.deepspeed_stage3_param_persistence_threshold,  # noqa: E501
-            "stage3_gather_16bit_weights_on_model_save": True,
             # zero3 offload cpu
             # "stage3_max_live_parameters": cfg.environment.deepspeed_stage3_max_live_parameters,  # noqa: E501
             # "stage3_max_reuse_distance": cfg.environment.deepspeed_stage3_max_reuse_distance,  # noqa: E501

From 4491c16d04ba89cd5fd8c03a6dd2d24f7c46b2f2 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Thu, 5 Oct 2023 03:46:19 -0700
Subject: [PATCH 45/52] remove unuse code

---
 llm_studio/src/utils/modeling_utils.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index 7ea3b842bb..e335671f86 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -99,8 +99,6 @@ def save_checkpoint(model: torch.nn.Module, path: str, cfg: Any):
     if cfg.environment.use_deepspeed:
         if path is not None:
             # gather model params from all ranks
-            # if hasattr(cfg.training, "lora") and cfg.training.lora:
-            #     model.backbone.save_pretrained(path)
             model.save_checkpoint(os.path.join(path, "ds_checkpoint"))
             if cfg.environment._local_rank == 0:
                 # load to cpu
@@ -114,8 +112,6 @@ def save_checkpoint(model: torch.nn.Module, path: str, cfg: Any):
     else:
         if cfg.environment._local_rank == 0:
             model = unwrap_model(model)
-            # if hasattr(cfg.training, "lora") and cfg.training.lora:
-            #     model.backbone.save_pretrained(path)
             checkpoint = {"model": model.state_dict()}
             if path is not None:
                 torch.save(checkpoint, os.path.join(path, "checkpoint.pth"))

From 9337741b2eecf25ad4440a2bd9785c2e495f0a74 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@users.noreply.github.com>
Date: Tue, 10 Oct 2023 16:32:09 +0000
Subject: [PATCH 46/52] Update requirements.txt

---
 requirements.txt | 29 ++++++++++++++++-------------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 468b1c97a2..9ca2907289 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,7 @@
 -i https://pypi.org/simple
 --extra-index-url https://download.pytorch.org/whl/cu118
 accelerate==0.23.0
-aiohttp==3.8.5 ; python_version >= '3.6'
+aiohttp==3.8.6 ; python_version >= '3.6'
 aiosignal==1.3.1 ; python_version >= '3.7'
 anyio==4.0.0 ; python_version >= '3.8'
 arrow==1.3.0 ; python_version >= '3.8'
@@ -12,11 +12,11 @@ azure-storage-blob==12.18.2 ; python_version >= '3.7'
 azure-storage-file-datalake==12.13.1
 beautifulsoup4==4.12.2
 bitsandbytes==0.41.1
-bleach==6.0.0 ; python_version >= '3.7'
+bleach==6.1.0 ; python_version >= '3.8'
 blessed==1.20.0 ; python_version >= '2.7'
-bokeh==3.2.2
-boto3==1.28.61
-botocore==1.31.61 ; python_version >= '3.7'
+bokeh==3.3.0
+boto3==1.28.62
+botocore==1.31.62 ; python_version >= '3.7'
 bravado==11.0.3 ; python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_full_version != '3.5.0'
 bravado-core==6.1.0 ; python_version >= '3.7'
 certifi==2023.7.22 ; python_version >= '3.6'
@@ -39,14 +39,15 @@ filelock==3.12.4 ; python_version >= '3.8'
 fqdn==1.5.1
 frozenlist==1.4.0 ; python_version >= '3.8'
 fsspec[http]==2023.6.0 ; python_version >= '3.8'
-future==0.18.3 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+future==0.18.3 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
 gitdb==4.0.10 ; python_version >= '3.7'
 gitpython==3.1.37 ; python_version >= '3.7'
 gputil==1.4.0
-greenlet==3.0.0 ; python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))
+greenlet==3.0.0 ; python_version >= '3' and (platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32'))))))
 h11==0.14.0 ; python_version >= '3.7'
 h2o-wave==0.26.2
 hf-transfer==0.1.3
+hjson==3.1.0
 httpcore==0.18.0 ; python_version >= '3.8'
 httpx==0.25.0 ; python_version >= '3.8'
 huggingface-hub==0.17.2
@@ -76,6 +77,7 @@ multidict==6.0.4 ; python_version >= '3.7'
 multiprocess==0.70.15 ; python_version >= '3.7'
 neptune==1.6.3
 networkx==3.1 ; python_version >= '3.8'
+ninja==1.11.1.1
 numpy==1.26.0
 oauthlib==3.2.2 ; python_version >= '3.6'
 openai==0.28.1
@@ -86,11 +88,12 @@ pillow==10.0.1 ; python_version >= '3.8'
 portalocker==2.8.2 ; python_version >= '3.8'
 protobuf==3.20.3
 psutil==5.9.5 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+py-cpuinfo==9.0.0
 pyarrow==13.0.0
 pycparser==2.21
-pydantic==1.10.12 ; python_version >= '3.7'
+pydantic==1.10.13 ; python_version >= '3.7'
 pyjwt==2.8.0 ; python_version >= '3.7'
-python-dateutil==2.8.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+python-dateutil==2.8.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
 python-editor==1.0.4
 python-slugify==8.0.1 ; python_version >= '3.7'
 pytz==2023.3.post1
@@ -111,8 +114,8 @@ scipy==1.11.3 ; python_version < '3.13' and python_version >= '3.9'
 secretstorage==3.3.3 ; sys_platform == 'linux'
 sentencepiece==0.1.99
 setuptools==68.2.2 ; python_version >= '3.8'
-simplejson==3.19.2 ; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'
-six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+simplejson==3.19.2 ; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2'
+six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
 smmap==5.0.1 ; python_version >= '3.7'
 sniffio==1.3.0 ; python_version >= '3.7'
 soupsieve==2.5 ; python_version >= '3.8'
@@ -138,12 +141,12 @@ types-python-dateutil==2.8.19.14
 typing-extensions==4.8.0 ; python_version >= '3.8'
 tzdata==2023.3 ; python_version >= '2'
 uri-template==1.3.0
-urllib3==1.26.17 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+urllib3==2.0.6 ; python_version >= '3.7'
 uvicorn==0.23.2 ; python_version >= '3.8'
 wcwidth==0.2.8
 webcolors==1.13
 webencodings==0.5.1
-websocket-client==1.6.3 ; python_version >= '3.8'
+websocket-client==1.6.4 ; python_version >= '3.8'
 xxhash==3.4.1 ; python_version >= '3.7'
 xyzservices==2023.10.0 ; python_version >= '3.8'
 yarl==1.9.2 ; python_version >= '3.7'

From 263f48aa0cf2d09e7c0111b9ad9f8729698dcb8f Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Wed, 18 Oct 2023 20:55:08 -0700
Subject: [PATCH 47/52] deepspeed==0.11.1

---
 Pipfile      |   2 +-
 Pipfile.lock | 526 ++++++++++++++++++++++++++-------------------------
 2 files changed, 270 insertions(+), 258 deletions(-)

diff --git a/Pipfile b/Pipfile
index 707afe8d5b..9543c1123d 100644
--- a/Pipfile
+++ b/Pipfile
@@ -50,7 +50,7 @@ tiktoken = "==0.5.1"
 hf-transfer = "==0.1.3"
 peft = "==0.5.0"
 azure-storage-file-datalake = ">=12.12.0"
-deepspeed = "==0.10.2"
+deepspeed = "==0.11.1"
 keyring = "==24.2.0"
 
 [dev-packages]
diff --git a/Pipfile.lock b/Pipfile.lock
index a65130ca69..c52b47b924 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "661181d44251f0bfa053c9c524c7117daf7a6402e819418780655eb059e7ff46"
+            "sha256": "d6da98ab1ca6c30933bd9ed0d472f64449b554aa87ed524c61c32974a497c71a"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -172,19 +172,19 @@
         },
         "azure-storage-blob": {
             "hashes": [
-                "sha256:e11935348981ffc005b848b55db25c04f2d1f90e1ee33000659906b763cf14c8",
-                "sha256:ffd864bf9abf33dfc72c6ef37899a19bd9d585a946a2c61e288b4420c035df3a"
+                "sha256:c278dde2ac41857a68d615c9f2b36d894ba877a7e84d62795603c7e79d0bb5e9",
+                "sha256:d8ced0deee3367fa3d4f3d1a03cd9edadf4440c0a371f503d623fa6c807554ee"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==12.18.2"
+            "version": "==12.18.3"
         },
         "azure-storage-file-datalake": {
             "hashes": [
-                "sha256:54e8b19e8e3d83bb5fbc51c3a17ed78d6be2811cc78af12beadc5b547bba574e",
-                "sha256:e53dbbfef817fac63de0df92787d8b84798d7f1e12d9d216f30b722b6b8ccf31"
+                "sha256:04c043bac0a996184eac043c42aecde880f9cebf2811af158f33e384458e8037",
+                "sha256:7b199828b1abe170d64d68d67e0b74200a9a445a2261c1a96fdd06f584d34f36"
             ],
             "index": "pypi",
-            "version": "==12.13.1"
+            "version": "==12.13.2"
         },
         "beautifulsoup4": {
             "hashes": [
@@ -228,19 +228,19 @@
         },
         "boto3": {
             "hashes": [
-                "sha256:0dfa2fc96ccafce4feb23044d6cba8b25075ad428a0c450d369d099c6a1059d2",
-                "sha256:148eeba0f1867b3db5b3e5ae2997d75a94d03fad46171374a0819168c36f7ed0"
+                "sha256:9d52a1605657aeb5b19b09cfc01d9a92f88a616a5daf5479a59656d6341ea6b3",
+                "sha256:ff3d0116e0ca6c096547652390025780eace3a28f6c04c9ffbf38448f1e5a87b"
             ],
             "index": "pypi",
-            "version": "==1.28.62"
+            "version": "==1.28.65"
         },
         "botocore": {
             "hashes": [
-                "sha256:272b78ac65256b6294cb9cdb0ac484d447ad3a85642e33cb6a3b1b8afee15a4c",
-                "sha256:be792d806afc064694a2d0b9b25779f3ca0c1584b29a35ac32e67f0064ddb8b7"
+                "sha256:90716c6f1af97e5c2f516e9a3379767ebdddcc6cbed79b026fa5038ce4e5e43e",
+                "sha256:f74e3da98dfcec17bc63ef58f82c643bf5bd7ec6cc11a26ede21cc4cd064917f"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.31.62"
+            "version": "==1.31.65"
         },
         "bravado": {
             "hashes": [
@@ -430,25 +430,25 @@
         },
         "cmake": {
             "hashes": [
-                "sha256:0ec349e938a75244ac6fbf0c5e4451ea61383d4f70173e6b09157722f4d60dde",
-                "sha256:149aaa0080f5fd1e180bed2486e602baafd8a7b1d5ac18219ff9e4378fba37e0",
-                "sha256:1edd401cc9a48562b920d3d0387130ddba4c85935f1ea777c64ba0423cb5009b",
-                "sha256:41c2aa92ce0accb43544a16e1d6bceb7f8900ac07def4ea7ba92ee95ba49be96",
-                "sha256:4c4cb267d9863281222c1f442e09c558cfac3e3454c45753ee0617692062a940",
-                "sha256:7e60dc95f78cf4ed538093d59a07744857ca344f4b1d0ef9e6508f47d0158a1d",
-                "sha256:82f6b547eb92218bb73ea5b91d7a852375d33b57bfc47f7421266ead081408e9",
-                "sha256:83f49328d37e777e711098064399d2666b243ae95d01aa275f1ad7323e973ab2",
-                "sha256:8a27928c8b387bd7df42083fe55d1490f964cb423d83861ec8d2170d2060d2c9",
-                "sha256:8ba53d322d5c84afd839138315d71ee9f892dcecb5e163e8d888f0d92cf43fe1",
-                "sha256:8c0ad28706b76d6b7c7643222d833af44bdd03da4371e3a6f0eac68e3a1d6366",
-                "sha256:94c22b97f230f4f13469215dafcf388595566a0075eaf12a527b6eb90c5c83f2",
-                "sha256:ac9b9290f67f0e7d9ccfb45396d5c6ea50dd12b46bfeb770f5a3051066fdca4e",
-                "sha256:b4f120e60823e793cc7d8645e75dce4519f76ce7692f2ab4b745e6cbca8b5345",
-                "sha256:daaa867afeeafabb1544fb129e9fbc3c537a7261430b6f00605268530f75b948",
-                "sha256:f289e26bd483e189933d1ee70987976d3fb88ce1ab6a8081c8da9d4fa44acec8",
-                "sha256:fff9dac42647c288d6c65f175de5ef0f158d3bc04eb354d6920af2c63b6d7e88"
-            ],
-            "version": "==3.27.6"
+                "sha256:009058bdf4f488709f38eaa5dd0ef0f89c6b9c6b6edd9d5b475a308ef75f80bb",
+                "sha256:1b9067ce0251cba3d4c018f2e1577ba9078e9c1eff6ad607ad5ce867843d4571",
+                "sha256:2fb48c780f1a6a3d19e785ebbb754be79d369e25a1cb81043fab049e709564da",
+                "sha256:499b38c77d52fb1964dbb38d0228fed246263a181939a8e753fde8ca227c8e1e",
+                "sha256:591f6b056527aefec009bc61a388776b2fc62444deb0038112a471031f61aeca",
+                "sha256:68983b09de633fc1ce6ab6bce9a25bfa181e41598e7c6bc0a6c0108773ee01cb",
+                "sha256:72289361866314f73be2ae63ddee224ff70223dcef9feb66d0072bf17e245564",
+                "sha256:7bf96237ba11ce2437dc5e071d96b510120a1be4708c631a64b2f38fb46bbd77",
+                "sha256:8056c99e371ff57229df2068364d7c32fea716cb53b4675f639edfb62663decf",
+                "sha256:810e592b606d05a3080a9c19ea839b13226f62cae447a22485b2365782f6b926",
+                "sha256:8bd1e1fa4fc8de7605c663d9408dceb649112f855aab05cca31fdb72e4d78364",
+                "sha256:9f4a7c7be2a25de5901f045618f41b833ea6c0f647115201d38e4fdf7e2815bc",
+                "sha256:b8a2fcb619b89d1cce7b52828316de9a1f27f0c90c2e39d1eae886428c8ee8c6",
+                "sha256:bd40d46dbad3555d5b3ce054bef24b85f256b19139493773751ab6f2b71c1219",
+                "sha256:bdbf0256f554f68c7b1d9740f5d059daf875b685c81a479cbe69038e84eb2fb9",
+                "sha256:c981aafcca2cd7210bd210ec75710c0f34e1fde1998cdcab812e4133e3ab615d",
+                "sha256:d582ef3e9ff0bd113581c1a32e881d1c2f9a34d2de76c93324a28593a76433db"
+            ],
+            "version": "==3.27.7"
         },
         "colorama": {
             "hashes": [
@@ -538,11 +538,14 @@
                 "sha256:2645f0e01735231e3c4d1568c95cc00ed477c1c2e2ed45cb123bef2ccbe72282",
                 "sha256:272ab041e95e4088321bffd63ead85d32f86f9fc79f37944663abb59e7fecbb1",
                 "sha256:2a96ff622f0db7f89d7c9aeadd9cc0c9bf61e804841a03a22ca919aa4955640d",
+                "sha256:2b5a463b01d8867f78a361defb4fadca63635965e8204f7e2409b717d85f0c1d",
+                "sha256:3cd45d09229960a89e0e3bbf71d29ac399391446f51279f3202846d7da106eb7",
                 "sha256:3d5ed0fa20b42e063ef66ad01d9948e868bbfc327bf86604e078b67f074f76f3",
                 "sha256:3f2e41dc8143d0e88ec9ba3ff66c8d2aea486b04931119abe9e394f9961d74bc",
                 "sha256:44e262f083a0d1c19bc7b8bf8aaa54d31653eab67762cdaeb68fb40b844d50b9",
                 "sha256:44fe99233ef2f42ff03d8395e7d97e0c45306eb356f6f01fa69bdb49783fdb8a",
                 "sha256:47ae97247a58d3095be800420bba7e43cc6958e67f9dfddd12decdb4c99c4d6f",
+                "sha256:4bb065d054e49645414bc84eaa49a6b7ed6c7185fb8ba2648c518344cf481144",
                 "sha256:4d56afb5f278a18743a218514825b6ab176f18a4084d8f6515c64e3acef19478",
                 "sha256:4f48e35701528beed092d97c5a1a6ba6c8902c3485d7e3a55c463162bb66afa0",
                 "sha256:53ac8df43546d3f70acbc5c17424a8c083418bd6d2cacfbd5108aaa8f3eb26db",
@@ -553,18 +556,24 @@
                 "sha256:60951e64d3e05ef2a46d2a92fc4e4563ae5e28bb3b6f231f2dca68a5078a72dc",
                 "sha256:66d6a7906fbe8c2e46e987d5a3a9cb235e931b3e4721ac7d8573151c419e3f73",
                 "sha256:6a8f0fd14bd11f2a625e5c554fd93c7775082c31ffd9fbabb4fe9db3031645d0",
+                "sha256:6bf4ec082a7647ed7463c5e0bd176850a9f013c258ad98f53055bcd1716c7fac",
                 "sha256:71bf6a6632648333c402a8692fe61f45416066eb0d8b7f4530cdf37fee221a11",
                 "sha256:79c36d95e89b43c29595c889c7a4d30d29aefc55d7c58a26a058b9bbe7abd5cf",
                 "sha256:7b0fa1f8600fdf093a4e5eac5dc56084047abec50d589a8121618e3fb1f9de3f",
                 "sha256:7e44dda432a8c8a47cb73869201d2f9777604bd913d859def84c659fb736cfd3",
                 "sha256:820421600d395d0e11b9da1e983acde453637090f5a15abb45f5182cd35f9c3a",
+                "sha256:831951a3eccebd5911387e3fff14483747d3dae5ad496afbd50447ffe2d03dba",
                 "sha256:89b683919ffc846b91b405d5f14df8a2ba1167f4ed9277150298fbe91a3d4aae",
+                "sha256:8e82276b120000834af3f776cbe22ac3016cd9e0ed00e7f3fb31e2ce95e17181",
                 "sha256:8ea1c781d3760df0b6ad80b7b19dc8e038e0638fb1cfabc68da96cedb8d0adca",
                 "sha256:91bb03ca0d3857f319afd13525d5c6214a55aa49778ce46a02c16f0eee37907c",
                 "sha256:9698ed7fe78b4de81dcee0624311ee93a4436f0b3e3c6b147a6745a3fd210272",
+                "sha256:9a3f45cd17730c00358643a814d5921a31c8a0d238582e08594fa86fdbc8401c",
+                "sha256:9f9e403be6307e98a47cc7d0132fe1fe4683c873c5d7d9da099fbac9c299a4b0",
                 "sha256:a08dcb7c7b54f82db4ee9120aaace06326499c0d4108770ee7ac63d7bd1d803d",
                 "sha256:a7f93316abc1abfd348b04afc6cadbbd4fba44cd91e7b9803c9330045a7a1885",
                 "sha256:aac9d49e16f473ceb4eaf74a53180eac3363127f01855c39122b400a988e80bf",
+                "sha256:b84565611f36ee8c3944b96bb01ee4d44c101acf84f1430c9414986ab3a7fb03",
                 "sha256:bee04fd1cdd5f2a2e91e4b271f22e228c698fe7b7f8ef209374d717f7889e80c",
                 "sha256:bef07e7d4607c4d70627e58eb630fe60e48b80a61ab05b33314e3296eb90af78",
                 "sha256:c2602c42ed101ada634fa37253d40946f0468b2b749689781cba715a7d78038e",
@@ -584,7 +593,8 @@
                 "sha256:f9340c70d95ea102cf51a07ecc09f93f1363e585d97b5276734a5f8c4476e560",
                 "sha256:fab800ed93fb129d85c63f26660b695fb194efb29765a163f269321778e28a8d",
                 "sha256:fbc59b223750a901d65d96333461ab17076594fa34448ed2ef911bd4b628f068",
-                "sha256:fbffb1f63edf4cb4272a25de288c2f2e20914bb93e003883656774e61794b960"
+                "sha256:fbffb1f63edf4cb4272a25de288c2f2e20914bb93e003883656774e61794b960",
+                "sha256:fd2e81c69baacb95fa28cdf844f714d7c03f0c805f4fa2decc5e9565e6b4405d"
             ],
             "markers": "python_version >= '3.7'",
             "version": "==2.7.0"
@@ -628,10 +638,10 @@
         },
         "deepspeed": {
             "hashes": [
-                "sha256:286bb1f01ea7340c9e18d78a562393dd1c12cc1406f1f5996f4be1d6984360ef"
+                "sha256:fa342f11f971e1bb002a537adfea6ac4c089b7a4da5aef5f2216a472cb35be7f"
             ],
             "index": "pypi",
-            "version": "==0.10.2"
+            "version": "==0.11.1"
         },
         "dill": {
             "hashes": [
@@ -806,11 +816,11 @@
         },
         "gitpython": {
             "hashes": [
-                "sha256:5f4c4187de49616d710a77e98ddf17b4782060a1788df441846bddefbb89ab33",
-                "sha256:f9b9ddc0761c125d5780eab2d64be4873fc6817c2899cbcb34b02344bdc7bc54"
+                "sha256:22b126e9ffb671fdd0c129796343a02bf67bf2994b35449ffc9321aa755e18a4",
+                "sha256:cf14627d5a8049ffbf49915732e5eddbe8134c3bdb9d476e6182b676fc573f8a"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.1.37"
+            "version": "==3.1.40"
         },
         "gputil": {
             "hashes": [
@@ -1089,9 +1099,9 @@
         },
         "lit": {
             "hashes": [
-                "sha256:d6a551eab550f81023c82a260cd484d63970d2be9fd7588111208e7d2ff62212"
+                "sha256:e6049032462be1e2928686cbd4a6cc5b3c545d83ecd078737fe79412c1f3fcc1"
             ],
-            "version": "==17.0.2"
+            "version": "==17.0.3"
         },
         "markupsafe": {
             "hashes": [
@@ -1383,41 +1393,41 @@
         },
         "numpy": {
             "hashes": [
-                "sha256:020cdbee66ed46b671429c7265cf00d8ac91c046901c55684954c3958525dab2",
-                "sha256:0621f7daf973d34d18b4e4bafb210bbaf1ef5e0100b5fa750bd9cde84c7ac292",
-                "sha256:0792824ce2f7ea0c82ed2e4fecc29bb86bee0567a080dacaf2e0a01fe7654369",
-                "sha256:09aaee96c2cbdea95de76ecb8a586cb687d281c881f5f17bfc0fb7f5890f6b91",
-                "sha256:166b36197e9debc4e384e9c652ba60c0bacc216d0fc89e78f973a9760b503388",
-                "sha256:186ba67fad3c60dbe8a3abff3b67a91351100f2661c8e2a80364ae6279720299",
-                "sha256:306545e234503a24fe9ae95ebf84d25cba1fdc27db971aa2d9f1ab6bba19a9dd",
-                "sha256:436c8e9a4bdeeee84e3e59614d38c3dbd3235838a877af8c211cfcac8a80b8d3",
-                "sha256:4a873a8180479bc829313e8d9798d5234dfacfc2e8a7ac188418189bb8eafbd2",
-                "sha256:4acc65dd65da28060e206c8f27a573455ed724e6179941edb19f97e58161bb69",
-                "sha256:51be5f8c349fdd1a5568e72713a21f518e7d6707bcf8503b528b88d33b57dc68",
-                "sha256:546b7dd7e22f3c6861463bebb000646fa730e55df5ee4a0224408b5694cc6148",
-                "sha256:5671338034b820c8d58c81ad1dafc0ed5a00771a82fccc71d6438df00302094b",
-                "sha256:637c58b468a69869258b8ae26f4a4c6ff8abffd4a8334c830ffb63e0feefe99a",
-                "sha256:767254ad364991ccfc4d81b8152912e53e103ec192d1bb4ea6b1f5a7117040be",
-                "sha256:7d484292eaeb3e84a51432a94f53578689ffdea3f90e10c8b203a99be5af57d8",
-                "sha256:7f6bad22a791226d0a5c7c27a80a20e11cfe09ad5ef9084d4d3fc4a299cca505",
-                "sha256:86f737708b366c36b76e953c46ba5827d8c27b7a8c9d0f471810728e5a2fe57c",
-                "sha256:8c6adc33561bd1d46f81131d5352348350fc23df4d742bb246cdfca606ea1208",
-                "sha256:914b28d3215e0c721dc75db3ad6d62f51f630cb0c277e6b3bcb39519bed10bd8",
-                "sha256:b44e6a09afc12952a7d2a58ca0a2429ee0d49a4f89d83a0a11052da696440e49",
-                "sha256:bb0d9a1aaf5f1cb7967320e80690a1d7ff69f1d47ebc5a9bea013e3a21faec95",
-                "sha256:c0b45c8b65b79337dee5134d038346d30e109e9e2e9d43464a2970e5c0e93229",
-                "sha256:c2e698cb0c6dda9372ea98a0344245ee65bdc1c9dd939cceed6bb91256837896",
-                "sha256:c78a22e95182fb2e7874712433eaa610478a3caf86f28c621708d35fa4fd6e7f",
-                "sha256:e062aa24638bb5018b7841977c360d2f5917268d125c833a686b7cbabbec496c",
-                "sha256:e5e18e5b14a7560d8acf1c596688f4dfd19b4f2945b245a71e5af4ddb7422feb",
-                "sha256:eae430ecf5794cb7ae7fa3808740b015aa80747e5266153128ef055975a72b99",
-                "sha256:ee84ca3c58fe48b8ddafdeb1db87388dce2c3c3f701bf447b05e4cfcc3679112",
-                "sha256:f042f66d0b4ae6d48e70e28d487376204d3cbf43b84c03bac57e28dac6151581",
-                "sha256:f8db2f125746e44dce707dd44d4f4efeea8d7e2b43aace3f8d1f235cfa2733dd",
-                "sha256:f93fc78fe8bf15afe2b8d6b6499f1c73953169fad1e9a8dd086cdff3190e7fdf"
+                "sha256:06934e1a22c54636a059215d6da99e23286424f316fddd979f5071093b648668",
+                "sha256:1c59c046c31a43310ad0199d6299e59f57a289e22f0f36951ced1c9eac3665b9",
+                "sha256:1d1bd82d539607951cac963388534da3b7ea0e18b149a53cf883d8f699178c0f",
+                "sha256:1e11668d6f756ca5ef534b5be8653d16c5352cbb210a5c2a79ff288e937010d5",
+                "sha256:3649d566e2fc067597125428db15d60eb42a4e0897fc48d28cb75dc2e0454e53",
+                "sha256:59227c981d43425ca5e5c01094d59eb14e8772ce6975d4b2fc1e106a833d5ae2",
+                "sha256:6081aed64714a18c72b168a9276095ef9155dd7888b9e74b5987808f0dd0a974",
+                "sha256:6965888d65d2848e8768824ca8288db0a81263c1efccec881cb35a0d805fcd2f",
+                "sha256:76ff661a867d9272cd2a99eed002470f46dbe0943a5ffd140f49be84f68ffc42",
+                "sha256:78ca54b2f9daffa5f323f34cdf21e1d9779a54073f0018a3094ab907938331a2",
+                "sha256:82e871307a6331b5f09efda3c22e03c095d957f04bf6bc1804f30048d0e5e7af",
+                "sha256:8ab9163ca8aeb7fd32fe93866490654d2f7dda4e61bc6297bf72ce07fdc02f67",
+                "sha256:9696aa2e35cc41e398a6d42d147cf326f8f9d81befcb399bc1ed7ffea339b64e",
+                "sha256:97e5d6a9f0702c2863aaabf19f0d1b6c2628fbe476438ce0b5ce06e83085064c",
+                "sha256:9f42284ebf91bdf32fafac29d29d4c07e5e9d1af862ea73686581773ef9e73a7",
+                "sha256:a03fb25610ef560a6201ff06df4f8105292ba56e7cdd196ea350d123fc32e24e",
+                "sha256:a5b411040beead47a228bde3b2241100454a6abde9df139ed087bd73fc0a4908",
+                "sha256:af22f3d8e228d84d1c0c44c1fbdeb80f97a15a0abe4f080960393a00db733b66",
+                "sha256:afd5ced4e5a96dac6725daeb5242a35494243f2239244fad10a90ce58b071d24",
+                "sha256:b9d45d1dbb9de84894cc50efece5b09939752a2d75aab3a8b0cef6f3a35ecd6b",
+                "sha256:bb894accfd16b867d8643fc2ba6c8617c78ba2828051e9a69511644ce86ce83e",
+                "sha256:c8c6c72d4a9f831f328efb1312642a1cafafaa88981d9ab76368d50d07d93cbe",
+                "sha256:cd7837b2b734ca72959a1caf3309457a318c934abef7a43a14bb984e574bbb9a",
+                "sha256:cdd9ec98f0063d93baeb01aad472a1a0840dee302842a2746a7a8e92968f9575",
+                "sha256:d1cfc92db6af1fd37a7bb58e55c8383b4aa1ba23d012bdbba26b4bcca45ac297",
+                "sha256:d1d2c6b7dd618c41e202c59c1413ef9b2c8e8a15f5039e344af64195459e3104",
+                "sha256:d2984cb6caaf05294b8466966627e80bf6c7afd273279077679cb010acb0e5ab",
+                "sha256:d58e8c51a7cf43090d124d5073bc29ab2755822181fcad978b12e144e5e5a4b3",
+                "sha256:d78f269e0c4fd365fc2992c00353e4530d274ba68f15e968d8bc3c69ce5f5244",
+                "sha256:dcfaf015b79d1f9f9c9fd0731a907407dc3e45769262d657d754c3a028586124",
+                "sha256:e44ccb93f30c75dfc0c3aa3ce38f33486a75ec9abadabd4e59f114994a9c4617",
+                "sha256:e509cbc488c735b43b5ffea175235cec24bbc57b227ef1acc691725beb230d1c"
             ],
             "index": "pypi",
-            "version": "==1.26.0"
+            "version": "==1.26.1"
         },
         "oauthlib": {
             "hashes": [
@@ -1484,63 +1494,63 @@
         },
         "pillow": {
             "hashes": [
-                "sha256:0462b1496505a3462d0f35dc1c4d7b54069747d65d00ef48e736acda2c8cbdff",
-                "sha256:186f7e04248103482ea6354af6d5bcedb62941ee08f7f788a1c7707bc720c66f",
-                "sha256:19e9adb3f22d4c416e7cd79b01375b17159d6990003633ff1d8377e21b7f1b21",
-                "sha256:28444cb6ad49726127d6b340217f0627abc8732f1194fd5352dec5e6a0105635",
-                "sha256:2872f2d7846cf39b3dbff64bc1104cc48c76145854256451d33c5faa55c04d1a",
-                "sha256:2cc6b86ece42a11f16f55fe8903595eff2b25e0358dec635d0a701ac9586588f",
-                "sha256:2d7e91b4379f7a76b31c2dda84ab9e20c6220488e50f7822e59dac36b0cd92b1",
-                "sha256:2fa6dd2661838c66f1a5473f3b49ab610c98a128fc08afbe81b91a1f0bf8c51d",
-                "sha256:32bec7423cdf25c9038fef614a853c9d25c07590e1a870ed471f47fb80b244db",
-                "sha256:3855447d98cced8670aaa63683808df905e956f00348732448b5a6df67ee5849",
-                "sha256:3a04359f308ebee571a3127fdb1bd01f88ba6f6fb6d087f8dd2e0d9bff43f2a7",
-                "sha256:3a0d3e54ab1df9df51b914b2233cf779a5a10dfd1ce339d0421748232cea9876",
-                "sha256:44e7e4587392953e5e251190a964675f61e4dae88d1e6edbe9f36d6243547ff3",
-                "sha256:459307cacdd4138edee3875bbe22a2492519e060660eaf378ba3b405d1c66317",
-                "sha256:4ce90f8a24e1c15465048959f1e94309dfef93af272633e8f37361b824532e91",
-                "sha256:50bd5f1ebafe9362ad622072a1d2f5850ecfa44303531ff14353a4059113b12d",
-                "sha256:522ff4ac3aaf839242c6f4e5b406634bfea002469656ae8358644fc6c4856a3b",
-                "sha256:552912dbca585b74d75279a7570dd29fa43b6d93594abb494ebb31ac19ace6bd",
-                "sha256:5d6c9049c6274c1bb565021367431ad04481ebb54872edecfcd6088d27edd6ed",
-                "sha256:697a06bdcedd473b35e50a7e7506b1d8ceb832dc238a336bd6f4f5aa91a4b500",
-                "sha256:71671503e3015da1b50bd18951e2f9daf5b6ffe36d16f1eb2c45711a301521a7",
-                "sha256:723bd25051454cea9990203405fa6b74e043ea76d4968166dfd2569b0210886a",
-                "sha256:764d2c0daf9c4d40ad12fbc0abd5da3af7f8aa11daf87e4fa1b834000f4b6b0a",
-                "sha256:787bb0169d2385a798888e1122c980c6eff26bf941a8ea79747d35d8f9210ca0",
-                "sha256:7f771e7219ff04b79e231d099c0a28ed83aa82af91fd5fa9fdb28f5b8d5addaf",
-                "sha256:847e8d1017c741c735d3cd1883fa7b03ded4f825a6e5fcb9378fd813edee995f",
-                "sha256:84efb46e8d881bb06b35d1d541aa87f574b58e87f781cbba8d200daa835b42e1",
-                "sha256:898f1d306298ff40dc1b9ca24824f0488f6f039bc0e25cfb549d3195ffa17088",
-                "sha256:8b451d6ead6e3500b6ce5c7916a43d8d8d25ad74b9102a629baccc0808c54971",
-                "sha256:8f06be50669087250f319b706decf69ca71fdecd829091a37cc89398ca4dc17a",
-                "sha256:92a23b0431941a33242b1f0ce6c88a952e09feeea9af4e8be48236a68ffe2205",
-                "sha256:93139acd8109edcdeffd85e3af8ae7d88b258b3a1e13a038f542b79b6d255c54",
-                "sha256:98533fd7fa764e5f85eebe56c8e4094db912ccbe6fbf3a58778d543cadd0db08",
-                "sha256:9f665d1e6474af9f9da5e86c2a3a2d2d6204e04d5af9c06b9d42afa6ebde3f21",
-                "sha256:b059ac2c4c7a97daafa7dc850b43b2d3667def858a4f112d1aa082e5c3d6cf7d",
-                "sha256:b1be1c872b9b5fcc229adeadbeb51422a9633abd847c0ff87dc4ef9bb184ae08",
-                "sha256:b7cf63d2c6928b51d35dfdbda6f2c1fddbe51a6bc4a9d4ee6ea0e11670dd981e",
-                "sha256:bc2e3069569ea9dbe88d6b8ea38f439a6aad8f6e7a6283a38edf61ddefb3a9bf",
-                "sha256:bcf1207e2f2385a576832af02702de104be71301c2696d0012b1b93fe34aaa5b",
-                "sha256:ca26ba5767888c84bf5a0c1a32f069e8204ce8c21d00a49c90dabeba00ce0145",
-                "sha256:cbe68deb8580462ca0d9eb56a81912f59eb4542e1ef8f987405e35a0179f4ea2",
-                "sha256:d6caf3cd38449ec3cd8a68b375e0c6fe4b6fd04edb6c9766b55ef84a6e8ddf2d",
-                "sha256:d72967b06be9300fed5cfbc8b5bafceec48bf7cdc7dab66b1d2549035287191d",
-                "sha256:d889b53ae2f030f756e61a7bff13684dcd77e9af8b10c6048fb2c559d6ed6eaf",
-                "sha256:de596695a75496deb3b499c8c4f8e60376e0516e1a774e7bc046f0f48cd620ad",
-                "sha256:e6a90167bcca1216606223a05e2cf991bb25b14695c518bc65639463d7db722d",
-                "sha256:ed2d9c0704f2dc4fa980b99d565c0c9a543fe5101c25b3d60488b8ba80f0cce1",
-                "sha256:ee7810cf7c83fa227ba9125de6084e5e8b08c59038a7b2c9045ef4dde61663b4",
-                "sha256:f0b4b06da13275bc02adfeb82643c4a6385bd08d26f03068c2796f60d125f6f2",
-                "sha256:f11c9102c56ffb9ca87134bd025a43d2aba3f1155f508eff88f694b33a9c6d19",
-                "sha256:f5bb289bb835f9fe1a1e9300d011eef4d69661bb9b34d5e196e5e82c4cb09b37",
-                "sha256:f6d3d4c905e26354e8f9d82548475c46d8e0889538cb0657aa9c6f0872a37aa4",
-                "sha256:fcb59711009b0168d6ee0bd8fb5eb259c4ab1717b2f538bbf36bacf207ef7a68",
-                "sha256:fd2a5403a75b54661182b75ec6132437a181209b901446ee5724b589af8edef1"
+                "sha256:00f438bb841382b15d7deb9a05cc946ee0f2c352653c7aa659e75e592f6fa17d",
+                "sha256:0248f86b3ea061e67817c47ecbe82c23f9dd5d5226200eb9090b3873d3ca32de",
+                "sha256:04f6f6149f266a100374ca3cc368b67fb27c4af9f1cc8cb6306d849dcdf12616",
+                "sha256:062a1610e3bc258bff2328ec43f34244fcec972ee0717200cb1425214fe5b839",
+                "sha256:0a026c188be3b443916179f5d04548092e253beb0c3e2ee0a4e2cdad72f66099",
+                "sha256:0f7c276c05a9767e877a0b4c5050c8bee6a6d960d7f0c11ebda6b99746068c2a",
+                "sha256:1a8413794b4ad9719346cd9306118450b7b00d9a15846451549314a58ac42219",
+                "sha256:1ab05f3db77e98f93964697c8efc49c7954b08dd61cff526b7f2531a22410106",
+                "sha256:1c3ac5423c8c1da5928aa12c6e258921956757d976405e9467c5f39d1d577a4b",
+                "sha256:1c41d960babf951e01a49c9746f92c5a7e0d939d1652d7ba30f6b3090f27e412",
+                "sha256:1fafabe50a6977ac70dfe829b2d5735fd54e190ab55259ec8aea4aaea412fa0b",
+                "sha256:1fb29c07478e6c06a46b867e43b0bcdb241b44cc52be9bc25ce5944eed4648e7",
+                "sha256:24fadc71218ad2b8ffe437b54876c9382b4a29e030a05a9879f615091f42ffc2",
+                "sha256:2cdc65a46e74514ce742c2013cd4a2d12e8553e3a2563c64879f7c7e4d28bce7",
+                "sha256:2ef6721c97894a7aa77723740a09547197533146fba8355e86d6d9a4a1056b14",
+                "sha256:3b834f4b16173e5b92ab6566f0473bfb09f939ba14b23b8da1f54fa63e4b623f",
+                "sha256:3d929a19f5469b3f4df33a3df2983db070ebb2088a1e145e18facbc28cae5b27",
+                "sha256:41f67248d92a5e0a2076d3517d8d4b1e41a97e2df10eb8f93106c89107f38b57",
+                "sha256:47e5bf85b80abc03be7455c95b6d6e4896a62f6541c1f2ce77a7d2bb832af262",
+                "sha256:4d0152565c6aa6ebbfb1e5d8624140a440f2b99bf7afaafbdbf6430426497f28",
+                "sha256:50d08cd0a2ecd2a8657bd3d82c71efd5a58edb04d9308185d66c3a5a5bed9610",
+                "sha256:61f1a9d247317fa08a308daaa8ee7b3f760ab1809ca2da14ecc88ae4257d6172",
+                "sha256:6932a7652464746fcb484f7fc3618e6503d2066d853f68a4bd97193a3996e273",
+                "sha256:7a7e3daa202beb61821c06d2517428e8e7c1aab08943e92ec9e5755c2fc9ba5e",
+                "sha256:7dbaa3c7de82ef37e7708521be41db5565004258ca76945ad74a8e998c30af8d",
+                "sha256:7df5608bc38bd37ef585ae9c38c9cd46d7c81498f086915b0f97255ea60c2818",
+                "sha256:806abdd8249ba3953c33742506fe414880bad78ac25cc9a9b1c6ae97bedd573f",
+                "sha256:883f216eac8712b83a63f41b76ddfb7b2afab1b74abbb413c5df6680f071a6b9",
+                "sha256:912e3812a1dbbc834da2b32299b124b5ddcb664ed354916fd1ed6f193f0e2d01",
+                "sha256:937bdc5a7f5343d1c97dc98149a0be7eb9704e937fe3dc7140e229ae4fc572a7",
+                "sha256:9882a7451c680c12f232a422730f986a1fcd808da0fd428f08b671237237d651",
+                "sha256:9a92109192b360634a4489c0c756364c0c3a2992906752165ecb50544c251312",
+                "sha256:9d7bc666bd8c5a4225e7ac71f2f9d12466ec555e89092728ea0f5c0c2422ea80",
+                "sha256:a5f63b5a68daedc54c7c3464508d8c12075e56dcfbd42f8c1bf40169061ae666",
+                "sha256:a646e48de237d860c36e0db37ecaecaa3619e6f3e9d5319e527ccbc8151df061",
+                "sha256:a89b8312d51715b510a4fe9fc13686283f376cfd5abca8cd1c65e4c76e21081b",
+                "sha256:a92386125e9ee90381c3369f57a2a50fa9e6aa8b1cf1d9c4b200d41a7dd8e992",
+                "sha256:ae88931f93214777c7a3aa0a8f92a683f83ecde27f65a45f95f22d289a69e593",
+                "sha256:afc8eef765d948543a4775f00b7b8c079b3321d6b675dde0d02afa2ee23000b4",
+                "sha256:b0eb01ca85b2361b09480784a7931fc648ed8b7836f01fb9241141b968feb1db",
+                "sha256:b1c25762197144e211efb5f4e8ad656f36c8d214d390585d1d21281f46d556ba",
+                "sha256:b4005fee46ed9be0b8fb42be0c20e79411533d1fd58edabebc0dd24626882cfd",
+                "sha256:b920e4d028f6442bea9a75b7491c063f0b9a3972520731ed26c83e254302eb1e",
+                "sha256:baada14941c83079bf84c037e2d8b7506ce201e92e3d2fa0d1303507a8538212",
+                "sha256:bb40c011447712d2e19cc261c82655f75f32cb724788df315ed992a4d65696bb",
+                "sha256:c0949b55eb607898e28eaccb525ab104b2d86542a85c74baf3a6dc24002edec2",
+                "sha256:c9aeea7b63edb7884b031a35305629a7593272b54f429a9869a4f63a1bf04c34",
+                "sha256:cfe96560c6ce2f4c07d6647af2d0f3c54cc33289894ebd88cfbb3bcd5391e256",
+                "sha256:d27b5997bdd2eb9fb199982bb7eb6164db0426904020dc38c10203187ae2ff2f",
+                "sha256:d921bc90b1defa55c9917ca6b6b71430e4286fc9e44c55ead78ca1a9f9eba5f2",
+                "sha256:e6bf8de6c36ed96c86ea3b6e1d5273c53f46ef518a062464cd7ef5dd2cf92e38",
+                "sha256:eaed6977fa73408b7b8a24e8b14e59e1668cfc0f4c40193ea7ced8e210adf996",
+                "sha256:fa1d323703cfdac2036af05191b969b910d8f115cf53093125e4058f62012c9a",
+                "sha256:fe1e26e1ffc38be097f0ba1d0d07fcade2bcfd1d023cda5b29935ae8052bd793"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==10.0.1"
+            "version": "==10.1.0"
         },
         "portalocker": {
             "hashes": [
@@ -1580,23 +1590,25 @@
         },
         "psutil": {
             "hashes": [
-                "sha256:104a5cc0e31baa2bcf67900be36acde157756b9c44017b86b2c049f11957887d",
-                "sha256:3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217",
-                "sha256:4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4",
-                "sha256:5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c",
-                "sha256:5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f",
-                "sha256:7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da",
-                "sha256:89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4",
-                "sha256:8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42",
-                "sha256:ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5",
-                "sha256:acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4",
-                "sha256:b258c0c1c9d145a1d5ceffab1134441c4c5113b2417fafff7315a917a026c3c9",
-                "sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f",
-                "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30",
-                "sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==5.9.5"
+                "sha256:10e8c17b4f898d64b121149afb136c53ea8b68c7531155147867b7b1ac9e7e28",
+                "sha256:18cd22c5db486f33998f37e2bb054cc62fd06646995285e02a51b1e08da97017",
+                "sha256:3ebf2158c16cc69db777e3c7decb3c0f43a7af94a60d72e87b2823aebac3d602",
+                "sha256:51dc3d54607c73148f63732c727856f5febec1c7c336f8f41fcbd6315cce76ac",
+                "sha256:6e5fb8dc711a514da83098bc5234264e551ad980cec5f85dabf4d38ed6f15e9a",
+                "sha256:70cb3beb98bc3fd5ac9ac617a327af7e7f826373ee64c80efd4eb2856e5051e9",
+                "sha256:748c9dd2583ed86347ed65d0035f45fa8c851e8d90354c122ab72319b5f366f4",
+                "sha256:91ecd2d9c00db9817a4b4192107cf6954addb5d9d67a969a4f436dbc9200f88c",
+                "sha256:92e0cc43c524834af53e9d3369245e6cc3b130e78e26100d1f63cdb0abeb3d3c",
+                "sha256:a6f01f03bf1843280f4ad16f4bde26b817847b4c1a0db59bf6419807bc5ce05c",
+                "sha256:c69596f9fc2f8acd574a12d5f8b7b1ba3765a641ea5d60fb4736bf3c08a8214a",
+                "sha256:ca2780f5e038379e520281e4c032dddd086906ddff9ef0d1b9dcf00710e5071c",
+                "sha256:daecbcbd29b289aac14ece28eca6a3e60aa361754cf6da3dfb20d4d32b6c7f57",
+                "sha256:e4b92ddcd7dd4cdd3f900180ea1e104932c7bce234fb88976e2a3b296441225a",
+                "sha256:fb8a697f11b0f5994550555fcfe3e69799e5b060c8ecf9e2f75c69302cc35c0d",
+                "sha256:ff18b8d1a784b810df0b0fff3bcb50ab941c3b8e2c8de5726f9c71c601c611aa"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
+            "version": "==5.9.6"
         },
         "py-cpuinfo": {
             "hashes": [
@@ -1928,108 +1940,108 @@
         },
         "rpds-py": {
             "hashes": [
-                "sha256:00a88003db3cc953f8656b59fc9af9d0637a1fb93c235814007988f8c153b2f2",
-                "sha256:049098dabfe705e9638c55a3321137a821399c50940041a6fcce267a22c70db2",
-                "sha256:08f07150c8ebbdbce1d2d51b8e9f4d588749a2af6a98035485ebe45c7ad9394e",
-                "sha256:125776d5db15162fdd9135372bef7fe4fb7c5f5810cf25898eb74a06a0816aec",
-                "sha256:13cbd79ccedc6b39c279af31ebfb0aec0467ad5d14641ddb15738bf6e4146157",
-                "sha256:18d5ff7fbd305a1d564273e9eb22de83ae3cd9cd6329fddc8f12f6428a711a6a",
-                "sha256:1c27942722cd5039bbf5098c7e21935a96243fed00ea11a9589f3c6c6424bd84",
-                "sha256:255a23bded80605e9f3997753e3a4b89c9aec9efb07ec036b1ca81440efcc1a9",
-                "sha256:2573ec23ad3a59dd2bc622befac845695972f3f2d08dc1a4405d017d20a6c225",
-                "sha256:2603e084054351cc65097da326570102c4c5bd07426ba8471ceaefdb0b642cc9",
-                "sha256:28b4942ec7d9d6114c1e08cace0157db92ef674636a38093cab779ace5742d3a",
-                "sha256:28e29dac59df890972f73c511948072897f512974714a803fe793635b80ff8c7",
-                "sha256:2a97406d5e08b7095428f01dac0d3c091dc072351151945a167e7968d2755559",
-                "sha256:2a9e864ec051a58fdb6bb2e6da03942adb20273897bc70067aee283e62bbac4d",
-                "sha256:2e0e2e01c5f61ddf47e3ed2d1fe1c9136e780ca6222d57a2517b9b02afd4710c",
-                "sha256:2e79eeeff8394284b09577f36316d410525e0cf0133abb3de10660e704d3d38e",
-                "sha256:2f2ac8bb01f705c5caaa7fe77ffd9b03f92f1b5061b94228f6ea5eaa0fca68ad",
-                "sha256:32819b662e3b4c26355a4403ea2f60c0a00db45b640fe722dd12db3d2ef807fb",
-                "sha256:3507c459767cf24c11e9520e2a37c89674266abe8e65453e5cb66398aa47ee7b",
-                "sha256:362faeae52dc6ccc50c0b6a01fa2ec0830bb61c292033f3749a46040b876f4ba",
-                "sha256:3650eae998dc718960e90120eb45d42bd57b18b21b10cb9ee05f91bff2345d48",
-                "sha256:36ff30385fb9fb3ac23a28bffdd4a230a5229ed5b15704b708b7c84bfb7fce51",
-                "sha256:3bc561c183684636c0099f9c3fbab8c1671841942edbce784bb01b4707d17924",
-                "sha256:3bd38b80491ef9686f719c1ad3d24d14fbd0e069988fdd4e7d1a6ffcdd7f4a13",
-                "sha256:3e37f1f134037601eb4b1f46854194f0cc082435dac2ee3de11e51529f7831f2",
-                "sha256:40f6e53461b19ddbb3354fe5bcf3d50d4333604ae4bf25b478333d83ca68002c",
-                "sha256:49db6c0a0e6626c2b97f5e7f8f7074da21cbd8ec73340c25e839a2457c007efa",
-                "sha256:4bcb1abecd998a72ad4e36a0fca93577fd0c059a6aacc44f16247031b98f6ff4",
-                "sha256:4cb55454a20d1b935f9eaab52e6ceab624a2efd8b52927c7ae7a43e02828dbe0",
-                "sha256:4f92d2372ec992c82fd7c74aa21e2a1910b3dcdc6a7e6392919a138f21d528a3",
-                "sha256:576d48e1e45c211e99fc02655ade65c32a75d3e383ccfd98ce59cece133ed02c",
-                "sha256:58bae860d1d116e6b4e1aad0cdc48a187d5893994f56d26db0c5534df7a47afd",
-                "sha256:5bb3f3cb6072c73e6ec1f865d8b80419b599f1597acf33f63fbf02252aab5a03",
-                "sha256:5db93f9017b384a4f194e1d89e1ce82d0a41b1fafdbbd3e0c8912baf13f2950f",
-                "sha256:5e41d5b334e8de4bc3f38843f31b2afa9a0c472ebf73119d3fd55cde08974bdf",
-                "sha256:60018626e637528a1fa64bb3a2b3e46ab7bf672052316d61c3629814d5e65052",
-                "sha256:6090ba604ea06b525a231450ae5d343917a393cbf50423900dea968daf61d16f",
-                "sha256:628fbb8be71a103499d10b189af7764996ab2634ed7b44b423f1e19901606e0e",
-                "sha256:6baea8a4f6f01e69e75cfdef3edd4a4d1c4b56238febbdf123ce96d09fbff010",
-                "sha256:6c5ca3eb817fb54bfd066740b64a2b31536eb8fe0b183dc35b09a7bd628ed680",
-                "sha256:70563a1596d2e0660ca2cebb738443437fc0e38597e7cbb276de0a7363924a52",
-                "sha256:7089d8bfa8064b28b2e39f5af7bf12d42f61caed884e35b9b4ea9e6fb1175077",
-                "sha256:72e9b1e92830c876cd49565d8404e4dcc9928302d348ea2517bc3f9e3a873a2a",
-                "sha256:7c7ca791bedda059e5195cf7c6b77384657a51429357cdd23e64ac1d4973d6dc",
-                "sha256:7f050ceffd8c730c1619a16bbf0b9cd037dcdb94b54710928ba38c7bde67e4a4",
-                "sha256:83da147124499fe41ed86edf34b4e81e951b3fe28edcc46288aac24e8a5c8484",
-                "sha256:86e8d6ff15fa7a9590c0addaf3ce52fb58bda4299cab2c2d0afa404db6848dab",
-                "sha256:8709eb4ab477c533b7d0a76cd3065d7d95c9e25e6b9f6e27caeeb8c63e8799c9",
-                "sha256:8e69bbe0ede8f7fe2616e779421bbdb37f025c802335a90f6416e4d98b368a37",
-                "sha256:8f90fc6dd505867514c8b8ef68a712dc0be90031a773c1ae2ad469f04062daef",
-                "sha256:9123ba0f3f98ff79780eebca9984a2b525f88563844b740f94cffb9099701230",
-                "sha256:927e3461dae0c09b1f2e0066e50c1a9204f8a64a3060f596e9a6742d3b307785",
-                "sha256:94876c21512535955a960f42a155213315e6ab06a4ce8ce372341a2a1b143eeb",
-                "sha256:98c0aecf661c175ce9cb17347fc51a5c98c3e9189ca57e8fcd9348dae18541db",
-                "sha256:9c7e7bd1fa1f535af71dfcd3700fc83a6dc261a1204f8f5327d8ffe82e52905d",
-                "sha256:9e7b3ad9f53ea9e085b3d27286dd13f8290969c0a153f8a52c8b5c46002c374b",
-                "sha256:9f9184744fb800c9f28e155a5896ecb54816296ee79d5d1978be6a2ae60f53c4",
-                "sha256:a3628815fd170a64624001bfb4e28946fd515bd672e68a1902d9e0290186eaf3",
-                "sha256:a5c330cb125983c5d380fef4a4155248a276297c86d64625fdaf500157e1981c",
-                "sha256:aa45cc71bf23a3181b8aa62466b5a2b7b7fb90fdc01df67ca433cd4fce7ec94d",
-                "sha256:aab24b9bbaa3d49e666e9309556591aa00748bd24ea74257a405f7fed9e8b10d",
-                "sha256:ac83f5228459b84fa6279e4126a53abfdd73cd9cc183947ee5084153880f65d7",
-                "sha256:ad21c60fc880204798f320387164dcacc25818a7b4ec2a0bf6b6c1d57b007d23",
-                "sha256:ae8a32ab77a84cc870bbfb60645851ca0f7d58fd251085ad67464b1445d632ca",
-                "sha256:b0f1d336786cb62613c72c00578c98e5bb8cd57b49c5bae5d4ab906ca7872f98",
-                "sha256:b28b9668a22ca2cfca4433441ba9acb2899624a323787a509a3dc5fbfa79c49d",
-                "sha256:b953d11b544ca5f2705bb77b177d8e17ab1bfd69e0fd99790a11549d2302258c",
-                "sha256:b9d8884d58ea8801e5906a491ab34af975091af76d1a389173db491ee7e316bb",
-                "sha256:ba3246c60303eab3d0e562addf25a983d60bddc36f4d1edc2510f056d19df255",
-                "sha256:bd0ad98c7d72b0e4cbfe89cdfa12cd07d2fd6ed22864341cdce12b318a383442",
-                "sha256:bf032367f921201deaecf221d4cc895ea84b3decf50a9c73ee106f961885a0ad",
-                "sha256:c31ecfc53ac03dad4928a1712f3a2893008bfba1b3cde49e1c14ff67faae2290",
-                "sha256:cbec8e43cace64e63398155dc585dc479a89fef1e57ead06c22d3441e1bd09c3",
-                "sha256:cc688a59c100f038fa9fec9e4ab457c2e2d1fca350fe7ea395016666f0d0a2dc",
-                "sha256:cd7da2adc721ccf19ac7ec86cae3a4fcaba03d9c477d5bd64ded6e9bb817bf3f",
-                "sha256:cd7e62e7d5bcfa38a62d8397fba6d0428b970ab7954c2197501cd1624f7f0bbb",
-                "sha256:d0f7f77a77c37159c9f417b8dd847f67a29e98c6acb52ee98fc6b91efbd1b2b6",
-                "sha256:d230fddc60caced271cc038e43e6fb8f4dd6b2dbaa44ac9763f2d76d05b0365a",
-                "sha256:d37f27ad80f742ef82796af3fe091888864958ad0bc8bab03da1830fa00c6004",
-                "sha256:d5ad7b1a1f6964d19b1a8acfc14bf7864f39587b3e25c16ca04f6cd1815026b3",
-                "sha256:d81359911c3bb31c899c6a5c23b403bdc0279215e5b3bc0d2a692489fed38632",
-                "sha256:d98802b78093c7083cc51f83da41a5be5a57d406798c9f69424bd75f8ae0812a",
-                "sha256:db0589e0bf41ff6ce284ab045ca89f27be1adf19e7bce26c2e7de6739a70c18b",
-                "sha256:ddbd113a37307638f94be5ae232a325155fd24dbfae2c56455da8724b471e7be",
-                "sha256:e3ece9aa6d07e18c966f14b4352a4c6f40249f6174d3d2c694c1062e19c6adbb",
-                "sha256:e3f9c9e5dd8eba4768e15f19044e1b5e216929a43a54b4ab329e103aed9f3eda",
-                "sha256:e41824343c2c129599645373992b1ce17720bb8a514f04ff9567031e1c26951e",
-                "sha256:e5dba1c11e089b526379e74f6c636202e4c5bad9a48c7416502b8a5b0d026c91",
-                "sha256:e791e3d13b14d0a7921804d0efe4d7bd15508bbcf8cb7a0c1ee1a27319a5f033",
-                "sha256:ec001689402b9104700b50a005c2d3d0218eae90eaa8bdbbd776fe78fe8a74b7",
-                "sha256:efffa359cc69840c8793f0c05a7b663de6afa7b9078fa6c80309ee38b9db677d",
-                "sha256:f1f191befea279cb9669b57be97ab1785781c8bab805900e95742ebfaa9cbf1d",
-                "sha256:f3331a3684192659fa1090bf2b448db928152fcba08222e58106f44758ef25f7",
-                "sha256:f40413d2859737ce6d95c29ce2dde0ef7cdc3063b5830ae4342fef5922c3bba7",
-                "sha256:f7ea49ddf51d5ec0c3cbd95190dd15e077a3153c8d4b22a33da43b5dd2b3c640",
-                "sha256:f82abb5c5b83dc30e96be99ce76239a030b62a73a13c64410e429660a5602bfd",
-                "sha256:fc20dadb102140dff63529e08ce6f9745dbd36e673ebb2b1c4a63e134bca81c2",
-                "sha256:fd37ab9a24021821b715478357af1cf369d5a42ac7405e83e5822be00732f463",
-                "sha256:ffd539d213c1ea2989ab92a5b9371ae7159c8c03cf2bcb9f2f594752f755ecd3"
+                "sha256:023574366002bf1bd751ebaf3e580aef4a468b3d3c216d2f3f7e16fdabd885ed",
+                "sha256:031f76fc87644a234883b51145e43985aa2d0c19b063e91d44379cd2786144f8",
+                "sha256:052a832078943d2b2627aea0d19381f607fe331cc0eb5df01991268253af8417",
+                "sha256:0699ab6b8c98df998c3eacf51a3b25864ca93dab157abe358af46dc95ecd9801",
+                "sha256:0713631d6e2d6c316c2f7b9320a34f44abb644fc487b77161d1724d883662e31",
+                "sha256:0774a46b38e70fdde0c6ded8d6d73115a7c39d7839a164cc833f170bbf539116",
+                "sha256:0898173249141ee99ffcd45e3829abe7bcee47d941af7434ccbf97717df020e5",
+                "sha256:09586f51a215d17efdb3a5f090d7cbf1633b7f3708f60a044757a5d48a83b393",
+                "sha256:102eac53bb0bf0f9a275b438e6cf6904904908562a1463a6fc3323cf47d7a532",
+                "sha256:10f32b53f424fc75ff7b713b2edb286fdbfc94bf16317890260a81c2c00385dc",
+                "sha256:150eec465dbc9cbca943c8e557a21afdcf9bab8aaabf386c44b794c2f94143d2",
+                "sha256:1d7360573f1e046cb3b0dceeb8864025aa78d98be4bb69f067ec1c40a9e2d9df",
+                "sha256:1f36a9d751f86455dc5278517e8b65580eeee37d61606183897f122c9e51cef3",
+                "sha256:24656dc36f866c33856baa3ab309da0b6a60f37d25d14be916bd3e79d9f3afcf",
+                "sha256:25860ed5c4e7f5e10c496ea78af46ae8d8468e0be745bd233bab9ca99bfd2647",
+                "sha256:26857f0f44f0e791f4a266595a7a09d21f6b589580ee0585f330aaccccb836e3",
+                "sha256:2bb2e4826be25e72013916eecd3d30f66fd076110de09f0e750163b416500721",
+                "sha256:2f6da6d842195fddc1cd34c3da8a40f6e99e4a113918faa5e60bf132f917c247",
+                "sha256:30adb75ecd7c2a52f5e76af50644b3e0b5ba036321c390b8e7ec1bb2a16dd43c",
+                "sha256:3339eca941568ed52d9ad0f1b8eb9fe0958fa245381747cecf2e9a78a5539c42",
+                "sha256:34ad87a831940521d462ac11f1774edf867c34172010f5390b2f06b85dcc6014",
+                "sha256:3777cc9dea0e6c464e4b24760664bd8831738cc582c1d8aacf1c3f546bef3f65",
+                "sha256:3953c6926a63f8ea5514644b7afb42659b505ece4183fdaaa8f61d978754349e",
+                "sha256:3c4eff26eddac49d52697a98ea01b0246e44ca82ab09354e94aae8823e8bda02",
+                "sha256:40578a6469e5d1df71b006936ce95804edb5df47b520c69cf5af264d462f2cbb",
+                "sha256:40f93086eef235623aa14dbddef1b9fb4b22b99454cb39a8d2e04c994fb9868c",
+                "sha256:4134aa2342f9b2ab6c33d5c172e40f9ef802c61bb9ca30d21782f6e035ed0043",
+                "sha256:442626328600bde1d09dc3bb00434f5374948838ce75c41a52152615689f9403",
+                "sha256:4a5ee600477b918ab345209eddafde9f91c0acd931f3776369585a1c55b04c57",
+                "sha256:4ce5a708d65a8dbf3748d2474b580d606b1b9f91b5c6ab2a316e0b0cf7a4ba50",
+                "sha256:516a611a2de12fbea70c78271e558f725c660ce38e0006f75139ba337d56b1f6",
+                "sha256:52c215eb46307c25f9fd2771cac8135d14b11a92ae48d17968eda5aa9aaf5071",
+                "sha256:53c43e10d398e365da2d4cc0bcaf0854b79b4c50ee9689652cdc72948e86f487",
+                "sha256:5752b761902cd15073a527b51de76bbae63d938dc7c5c4ad1e7d8df10e765138",
+                "sha256:5e8a78bd4879bff82daef48c14d5d4057f6856149094848c3ed0ecaf49f5aec2",
+                "sha256:5ed505ec6305abd2c2c9586a7b04fbd4baf42d4d684a9c12ec6110deefe2a063",
+                "sha256:5ee97c683eaface61d38ec9a489e353d36444cdebb128a27fe486a291647aff6",
+                "sha256:61fa268da6e2e1cd350739bb61011121fa550aa2545762e3dc02ea177ee4de35",
+                "sha256:64ccc28683666672d7c166ed465c09cee36e306c156e787acef3c0c62f90da5a",
+                "sha256:66414dafe4326bca200e165c2e789976cab2587ec71beb80f59f4796b786a238",
+                "sha256:68fe9199184c18d997d2e4293b34327c0009a78599ce703e15cd9a0f47349bba",
+                "sha256:6a555ae3d2e61118a9d3e549737bb4a56ff0cec88a22bd1dfcad5b4e04759175",
+                "sha256:6bdc11f9623870d75692cc33c59804b5a18d7b8a4b79ef0b00b773a27397d1f6",
+                "sha256:6cf4393c7b41abbf07c88eb83e8af5013606b1cdb7f6bc96b1b3536b53a574b8",
+                "sha256:6eef672de005736a6efd565577101277db6057f65640a813de6c2707dc69f396",
+                "sha256:734c41f9f57cc28658d98270d3436dba65bed0cfc730d115b290e970150c540d",
+                "sha256:73e0a78a9b843b8c2128028864901f55190401ba38aae685350cf69b98d9f7c9",
+                "sha256:775049dfa63fb58293990fc59473e659fcafd953bba1d00fc5f0631a8fd61977",
+                "sha256:7854a207ef77319ec457c1eb79c361b48807d252d94348305db4f4b62f40f7f3",
+                "sha256:78ca33811e1d95cac8c2e49cb86c0fb71f4d8409d8cbea0cb495b6dbddb30a55",
+                "sha256:79edd779cfc46b2e15b0830eecd8b4b93f1a96649bcb502453df471a54ce7977",
+                "sha256:7bf347b495b197992efc81a7408e9a83b931b2f056728529956a4d0858608b80",
+                "sha256:7fde6d0e00b2fd0dbbb40c0eeec463ef147819f23725eda58105ba9ca48744f4",
+                "sha256:81de24a1c51cfb32e1fbf018ab0bdbc79c04c035986526f76c33e3f9e0f3356c",
+                "sha256:879fb24304ead6b62dbe5034e7b644b71def53c70e19363f3c3be2705c17a3b4",
+                "sha256:8e7f2219cb72474571974d29a191714d822e58be1eb171f229732bc6fdedf0ac",
+                "sha256:9164ec8010327ab9af931d7ccd12ab8d8b5dc2f4c6a16cbdd9d087861eaaefa1",
+                "sha256:945eb4b6bb8144909b203a88a35e0a03d22b57aefb06c9b26c6e16d72e5eb0f0",
+                "sha256:99a57006b4ec39dbfb3ed67e5b27192792ffb0553206a107e4aadb39c5004cd5",
+                "sha256:9e9184fa6c52a74a5521e3e87badbf9692549c0fcced47443585876fcc47e469",
+                "sha256:9ff93d3aedef11f9c4540cf347f8bb135dd9323a2fc705633d83210d464c579d",
+                "sha256:a360cfd0881d36c6dc271992ce1eda65dba5e9368575663de993eeb4523d895f",
+                "sha256:a5d7ed104d158c0042a6a73799cf0eb576dfd5fc1ace9c47996e52320c37cb7c",
+                "sha256:ac17044876e64a8ea20ab132080ddc73b895b4abe9976e263b0e30ee5be7b9c2",
+                "sha256:ad857f42831e5b8d41a32437f88d86ead6c191455a3499c4b6d15e007936d4cf",
+                "sha256:b2039f8d545f20c4e52713eea51a275e62153ee96c8035a32b2abb772b6fc9e5",
+                "sha256:b455492cab07107bfe8711e20cd920cc96003e0da3c1f91297235b1603d2aca7",
+                "sha256:b4a9fe992887ac68256c930a2011255bae0bf5ec837475bc6f7edd7c8dfa254e",
+                "sha256:b5a53f5998b4bbff1cb2e967e66ab2addc67326a274567697379dd1e326bded7",
+                "sha256:b788276a3c114e9f51e257f2a6f544c32c02dab4aa7a5816b96444e3f9ffc336",
+                "sha256:bddd4f91eede9ca5275e70479ed3656e76c8cdaaa1b354e544cbcf94c6fc8ac4",
+                "sha256:c0503c5b681566e8b722fe8c4c47cce5c7a51f6935d5c7012c4aefe952a35eed",
+                "sha256:c1b3cd23d905589cb205710b3988fc8f46d4a198cf12862887b09d7aaa6bf9b9",
+                "sha256:c48f3fbc3e92c7dd6681a258d22f23adc2eb183c8cb1557d2fcc5a024e80b094",
+                "sha256:c63c3ef43f0b3fb00571cff6c3967cc261c0ebd14a0a134a12e83bdb8f49f21f",
+                "sha256:c6c45a2d2b68c51fe3d9352733fe048291e483376c94f7723458cfd7b473136b",
+                "sha256:caa1afc70a02645809c744eefb7d6ee8fef7e2fad170ffdeacca267fd2674f13",
+                "sha256:cc435d059f926fdc5b05822b1be4ff2a3a040f3ae0a7bbbe672babb468944722",
+                "sha256:cf693eb4a08eccc1a1b636e4392322582db2a47470d52e824b25eca7a3977b53",
+                "sha256:cf71343646756a072b85f228d35b1d7407da1669a3de3cf47f8bbafe0c8183a4",
+                "sha256:d08f63561c8a695afec4975fae445245386d645e3e446e6f260e81663bfd2e38",
+                "sha256:d29ddefeab1791e3c751e0189d5f4b3dbc0bbe033b06e9c333dca1f99e1d523e",
+                "sha256:d7f5e15c953ace2e8dde9824bdab4bec50adb91a5663df08d7d994240ae6fa31",
+                "sha256:d858532212f0650be12b6042ff4378dc2efbb7792a286bee4489eaa7ba010586",
+                "sha256:d97dd44683802000277bbf142fd9f6b271746b4846d0acaf0cefa6b2eaf2a7ad",
+                "sha256:dcdc88b6b01015da066da3fb76545e8bb9a6880a5ebf89e0f0b2e3ca557b3ab7",
+                "sha256:dd609fafdcdde6e67a139898196698af37438b035b25ad63704fd9097d9a3482",
+                "sha256:defa2c0c68734f4a82028c26bcc85e6b92cced99866af118cd6a89b734ad8e0d",
+                "sha256:e22260a4741a0e7a206e175232867b48a16e0401ef5bce3c67ca5b9705879066",
+                "sha256:e225a6a14ecf44499aadea165299092ab0cba918bb9ccd9304eab1138844490b",
+                "sha256:e3df0bc35e746cce42579826b89579d13fd27c3d5319a6afca9893a9b784ff1b",
+                "sha256:e6fcc026a3f27c1282c7ed24b7fcac82cdd70a0e84cc848c0841a3ab1e3dea2d",
+                "sha256:e782379c2028a3611285a795b89b99a52722946d19fc06f002f8b53e3ea26ea9",
+                "sha256:e8cdd52744f680346ff8c1ecdad5f4d11117e1724d4f4e1874f3a67598821069",
+                "sha256:e9616f5bd2595f7f4a04b67039d890348ab826e943a9bfdbe4938d0eba606971",
+                "sha256:e98c4c07ee4c4b3acf787e91b27688409d918212dfd34c872201273fdd5a0e18",
+                "sha256:ebdab79f42c5961682654b851f3f0fc68e6cc7cd8727c2ac4ffff955154123c1",
+                "sha256:f0f17f2ce0f3529177a5fff5525204fad7b43dd437d017dd0317f2746773443d",
+                "sha256:f4e56860a5af16a0fcfa070a0a20c42fbb2012eed1eb5ceeddcc7f8079214281"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.10.4"
+            "version": "==0.10.6"
         },
         "s3transfer": {
             "hashes": [
@@ -2797,11 +2809,11 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:7a7c7003b000adf9e7ca2a377c9688bbc54ed41b985789ed576570342a375cd2",
-                "sha256:b19e1a85d206b56d7df1d5e683df4a7725252a964e3993648dd0fb5a1c157564"
+                "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84",
+                "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.0.6"
+            "version": "==2.0.7"
         },
         "uvicorn": {
             "hashes": [
@@ -3358,11 +3370,11 @@
         },
         "pycodestyle": {
             "hashes": [
-                "sha256:259bcc17857d8a8b3b4a2327324b79e5f020a13c16074670f9c8c8f872ea76d0",
-                "sha256:5d1013ba8dc7895b548be5afb05740ca82454fd899971563d2ef625d090326f8"
+                "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f",
+                "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.11.0"
+            "version": "==2.11.1"
         },
         "pyflakes": {
             "hashes": [
@@ -3429,11 +3441,11 @@
         },
         "types-requests": {
             "hashes": [
-                "sha256:39894cbca3fb3d032ed8bdd02275b4273471aa5668564617cc1734b0a65ffdf8",
-                "sha256:e1b325c687b3494a2f528ab06e411d7092cc546cc9245c000bacc2fca5ae96d4"
+                "sha256:b32b9a86beffa876c0c3ac99a4cd3b8b51e973fb8e3bd4e0a6bb32c7efad80fc",
+                "sha256:dc5852a76f1eaf60eafa81a2e50aefa3d1f015c34cf0cba130930866b1b22a92"
             ],
             "index": "pypi",
-            "version": "==2.31.0.8"
+            "version": "==2.31.0.10"
         },
         "types-toml": {
             "hashes": [
@@ -3453,11 +3465,11 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:7a7c7003b000adf9e7ca2a377c9688bbc54ed41b985789ed576570342a375cd2",
-                "sha256:b19e1a85d206b56d7df1d5e683df4a7725252a964e3993648dd0fb5a1c157564"
+                "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84",
+                "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.0.6"
+            "version": "==2.0.7"
         },
         "wheel": {
             "hashes": [

From 882631aa8bd5546b5cfaa043de34c480f37e7d0d Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@users.noreply.github.com>
Date: Thu, 19 Oct 2023 03:59:08 +0000
Subject: [PATCH 48/52] Update requirements.txt

---
 requirements.txt | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 9ca2907289..61e7b44ed1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,29 +8,29 @@ arrow==1.3.0 ; python_version >= '3.8'
 async-timeout==4.0.3 ; python_version >= '3.7'
 attrs==23.1.0 ; python_version >= '3.7'
 azure-core==1.29.4 ; python_version >= '3.7'
-azure-storage-blob==12.18.2 ; python_version >= '3.7'
-azure-storage-file-datalake==12.13.1
+azure-storage-blob==12.18.3 ; python_version >= '3.7'
+azure-storage-file-datalake==12.13.2
 beautifulsoup4==4.12.2
 bitsandbytes==0.41.1
 bleach==6.1.0 ; python_version >= '3.8'
 blessed==1.20.0 ; python_version >= '2.7'
 bokeh==3.3.0
-boto3==1.28.62
-botocore==1.31.62 ; python_version >= '3.7'
+boto3==1.28.65
+botocore==1.31.65 ; python_version >= '3.7'
 bravado==11.0.3 ; python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_full_version != '3.5.0'
 bravado-core==6.1.0 ; python_version >= '3.7'
 certifi==2023.7.22 ; python_version >= '3.6'
 cffi==1.16.0 ; python_version >= '3.8'
 charset-normalizer==3.3.0 ; python_full_version >= '3.7.0'
 click==8.1.7 ; python_version >= '3.7'
-cmake==3.27.6
+cmake==3.27.7
 colorama==0.4.6 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
 contourpy==1.1.1 ; python_version >= '3.8'
 coolname==2.2.0
 cramjam==2.7.0 ; python_version >= '3.7'
 cryptography==41.0.4 ; python_version >= '3.7'
 datasets==2.14.5
-deepspeed==0.10.2
+deepspeed==0.11.1
 dill==0.3.7
 einops==0.6.1
 exceptiongroup==1.1.3 ; python_version < '3.11'
@@ -41,7 +41,7 @@ frozenlist==1.4.0 ; python_version >= '3.8'
 fsspec[http]==2023.6.0 ; python_version >= '3.8'
 future==0.18.3 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
 gitdb==4.0.10 ; python_version >= '3.7'
-gitpython==3.1.37 ; python_version >= '3.7'
+gitpython==3.1.40 ; python_version >= '3.7'
 gputil==1.4.0
 greenlet==3.0.0 ; python_version >= '3' and (platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32'))))))
 h11==0.14.0 ; python_version >= '3.7'
@@ -67,7 +67,7 @@ jsonschema==4.19.1 ; python_version >= '3.8'
 jsonschema-specifications==2023.7.1 ; python_version >= '3.8'
 kaggle==1.5.16
 keyring==24.2.0
-lit==17.0.2
+lit==17.0.3
 markupsafe==2.1.3 ; python_version >= '3.7'
 monotonic==1.6
 more-itertools==10.1.0 ; python_version >= '3.8'
@@ -78,16 +78,16 @@ multiprocess==0.70.15 ; python_version >= '3.7'
 neptune==1.6.3
 networkx==3.1 ; python_version >= '3.8'
 ninja==1.11.1.1
-numpy==1.26.0
+numpy==1.26.1
 oauthlib==3.2.2 ; python_version >= '3.6'
 openai==0.28.1
 packaging==23.2 ; python_version >= '3.7'
 pandas==2.1.1
 peft==0.5.0
-pillow==10.0.1 ; python_version >= '3.8'
+pillow==10.1.0 ; python_version >= '3.8'
 portalocker==2.8.2 ; python_version >= '3.8'
 protobuf==3.20.3
-psutil==5.9.5 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+psutil==5.9.6 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
 py-cpuinfo==9.0.0
 pyarrow==13.0.0
 pycparser==2.21
@@ -105,7 +105,7 @@ requests==2.31.0 ; python_version >= '3.7'
 requests-oauthlib==1.3.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 rfc3339-validator==0.1.4
 rfc3987==1.3.8
-rpds-py==0.10.4 ; python_version >= '3.8'
+rpds-py==0.10.6 ; python_version >= '3.8'
 s3transfer==0.7.0 ; python_version >= '3.7'
 sacrebleu==2.0.0
 safetensors==0.4.0 ; python_version >= '3.7'
@@ -141,7 +141,7 @@ types-python-dateutil==2.8.19.14
 typing-extensions==4.8.0 ; python_version >= '3.8'
 tzdata==2023.3 ; python_version >= '2'
 uri-template==1.3.0
-urllib3==2.0.6 ; python_version >= '3.7'
+urllib3==2.0.7 ; python_version >= '3.7'
 uvicorn==0.23.2 ; python_version >= '3.8'
 wcwidth==0.2.8
 webcolors==1.13

From 368f0af0f5c022a6a63d8a8589a5d301c39929a3 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Thu, 19 Oct 2023 23:53:27 -0700
Subject: [PATCH 49/52] temp fix for deepspeed slow gen

---
 train.py | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/train.py b/train.py
index 77e2b30ffc..bf856a1225 100644
--- a/train.py
+++ b/train.py
@@ -355,9 +355,21 @@ def run_train(
                 if cfg.training.evaluation_epochs == 1:
                     progress_bar.close()
 
+                # TODO: Move back after fixing slow generation of deepspeed.
+                if not cfg.training.save_best_checkpoint:
+                    checkpoint_path = cfg.output_directory
+                    if cfg.environment._local_rank == 0:
+                        logger.info(
+                            f"Saving last model checkpoint: "
+                            f"val_loss {val_loss:.5}, val_{cfg.prediction.metric} "
+                            f"{val_metric:.5} to {checkpoint_path}"
+                        )
+                    save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
+
                 val_loss, val_metric = run_eval(
                     cfg=cfg, model=model, val_dataloader=val_dataloader, val_df=val_df
                 )
+
                 if cfg.training.save_best_checkpoint:
                     if objective_op(val_metric, best_val_metric):
                         checkpoint_path = cfg.output_directory
@@ -369,15 +381,6 @@ def run_train(
                             )
                         save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
                         best_val_metric = val_metric
-                else:
-                    checkpoint_path = cfg.output_directory
-                    if cfg.environment._local_rank == 0:
-                        logger.info(
-                            f"Saving last model checkpoint: "
-                            f"val_loss {val_loss:.5}, val_{cfg.prediction.metric} "
-                            f"{val_metric:.5} to {checkpoint_path}"
-                        )
-                    save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
 
                 model.train()
 

From d5dbbfbde2cb080da6b06511c3e34377d56c9eac Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Fri, 20 Oct 2023 00:01:30 -0700
Subject: [PATCH 50/52] style

---
 llm_studio/src/utils/modeling_utils.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index 5c52512268..850f1bfaf6 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -19,7 +19,6 @@
     MixedPrecision,
 )
 from torch.nn.parallel import DistributedDataParallel
-from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm
 from transformers import (
     AutoConfig,
@@ -296,7 +295,7 @@ def wrap_model_distributed(
                 val_dataloader.dataset,
                 num_replicas=cfg.environment._world_size,
                 rank=cfg.environment._local_rank,
-            )
+            ),
         )
     else:
         find_unused_parameters = cfg.environment.find_unused_parameters

From 5b8499ca727fa23812b2ce77ed15b5c0a5ab1823 Mon Sep 17 00:00:00 2001
From: haqishen <haqishen@gmail.com>
Date: Fri, 20 Oct 2023 00:11:28 -0700
Subject: [PATCH 51/52] style

---
 train.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/train.py b/train.py
index 069e659d0f..c942734c5b 100644
--- a/train.py
+++ b/train.py
@@ -972,9 +972,7 @@ def run(cfg: Any) -> None:
         save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
 
     if cfg.environment._local_rank == 0:
-
         save_config_yaml(f"{cfg.output_directory}/cfg.yaml", cfg)
-
         save_prediction_outputs(cfg.experiment_name, experiment_path)
 
         flag_path = os.path.join(cfg.output_directory, "flags.json")

From 07bb4b233eedf17c377ba4a9b203438e32b3635e Mon Sep 17 00:00:00 2001
From: Philipp Singer <killver@gmail.com>
Date: Tue, 24 Oct 2023 07:19:04 +0000
Subject: [PATCH 52/52] fix

---
 train.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/train.py b/train.py
index c942734c5b..b00a789af7 100644
--- a/train.py
+++ b/train.py
@@ -360,9 +360,7 @@ def run_train(
                     checkpoint_path = cfg.output_directory
                     if cfg.environment._local_rank == 0:
                         logger.info(
-                            f"Saving last model checkpoint: "
-                            f"val_loss {val_loss:.5}, val_{cfg.prediction.metric} "
-                            f"{val_metric:.5} to {checkpoint_path}"
+                            f"Saving last model checkpoint to {checkpoint_path}"
                         )
                     save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)