From e751eb4c3b1e81eae3247b24230e327642f63c8d Mon Sep 17 00:00:00 2001
From: Ritchie Ng <ritchieng@u.nus.edu>
Date: Sun, 10 Mar 2024 17:08:41 +0800
Subject: [PATCH] Deployed d9018d4 with MkDocs version: 1.5.3

---
 404.html                                      |  188 +-
 about/index.html                              |  190 +-
 assets/javascripts/bundle.c8d2eff1.min.js     |   29 +
 assets/javascripts/bundle.c8d2eff1.min.js.map |    7 +
 assets/javascripts/bundle.febc23d1.min.js     |   29 -
 assets/javascripts/bundle.febc23d1.min.js.map |    7 -
 assets/javascripts/lunr/min/lunr.el.min.js    |    1 +
 assets/javascripts/lunr/min/lunr.he.min.js    |    1 +
 assets/javascripts/lunr/min/lunr.hy.min.js    |    1 +
 assets/javascripts/lunr/min/lunr.kn.min.js    |    1 +
 assets/javascripts/lunr/min/lunr.ko.min.js    |    1 +
 assets/javascripts/lunr/min/lunr.sa.min.js    |    1 +
 assets/javascripts/lunr/min/lunr.ta.min.js    |    1 +
 assets/javascripts/lunr/min/lunr.te.min.js    |    1 +
 .../workers/search.b8dbb3d2.min.js            |   42 +
 .../workers/search.b8dbb3d2.min.js.map        |    7 +
 assets/stylesheets/main.7e359304.min.css      |    1 +
 assets/stylesheets/main.7e359304.min.css.map  |    1 +
 assets/stylesheets/palette.06af60db.min.css   |    1 +
 .../stylesheets/palette.06af60db.min.css.map  |    1 +
 consultancy/index.html                        |  188 +-
 .../nosql/cassandra/intro/index.html          |  190 +-
 .../cassandra/setting_up_cluster/index.html   |  188 +-
 .../derivative_gradient_jacobian/index.html   |  188 +-
 .../index.html                                |  192 +-
 .../lr_scheduling/index.html                  |  188 +-
 .../optimizers/index.html                     |  188 +-
 .../index.html                                |  212 +-
 deep_learning/course_progression/index.html   |  238 +-
 .../bellman_mdp/index.html                    |  188 +-
 .../dynamic_programming_frozenlake/index.html |  196 +-
 .../rl_notes/index.html                       |  188 +-
 .../supervised_to_rl/index.html               |  188 +-
 .../fromscratch/fromscratch_cnn/index.html    |  200 +-
 .../index.html                                |  192 +-
 deep_learning/intro/index.html                |  190 +-
 .../pytorch_autoencoder/index.html            |  188 +-
 .../index.html                                |  236 +-
 .../pytorch_fc_overcomplete_ae/index.html     |  188 +-
 .../index.html                                |  188 +-
 .../pytorch_gradients/index.html              |  188 +-
 .../pytorch_linear_regression/index.html      |  188 +-
 .../pytorch_logistic_regression/index.html    |  188 +-
 .../pytorch_lstm_neuralnetwork/index.html     |  188 +-
 .../pytorch_matrices/index.html               |  216 +-
 .../index.html                                |  188 +-
 .../index.html                                |  190 +-
 deep_learning/readings/index.html             |  188 +-
 home/index.html                               |  190 +-
 index.html                                    |  202 +-
 .../hpc_containers_apptainer/index.html       | 2344 ++++++++++++++++
 language_model/intro/index.html               | 1747 ++++++++++++
 .../index.html                                | 2455 +++++++++++++++++
 .../gpu_fractional_differencing/index.html    |  202 +-
 machine_learning/gpu/rapids_cudf/index.html   |  192 +-
 machine_learning/intro/index.html             |  198 +-
 .../index.html                                |  188 +-
 .../index.html                                |  188 +-
 .../index.html                                |  192 +-
 .../index.html                                |  188 +-
 .../index.html                                |  188 +-
 .../index.html                                |  188 +-
 .../index.html                                |  188 +-
 .../index.html                                |  188 +-
 news/it_youth_leader_2019_03_11/index.html    |  188 +-
 .../index.html                                |  188 +-
 news/news/index.html                          |  190 +-
 .../index.html                                |  188 +-
 .../index.html                                |  188 +-
 ood/intro/index.html                          |  190 +-
 pipeline/index.html                           |  188 +-
 programming/bash/bash/index.html              |  336 ++-
 programming/cpp/cpp/index.html                |  604 ++--
 programming/electron/electron/index.html      |  332 ++-
 programming/intro/index.html                  |  190 +-
 programming/javascript/javascript/index.html  |  194 +-
 .../numpycupy/linalg_numpy_cupy/index.html    |  188 +-
 programming/plotting/intro/index.html         |  190 +-
 programming/python/python/index.html          |  188 +-
 programming/r/r/index.html                    |  318 ++-
 programming/sympy/calculus_sympy/index.html   |  188 +-
 review/index.html                             |  188 +-
 search/search_index.json                      |    2 +-
 sitemap.xml                                   |  135 +-
 sitemap.xml.gz                                |  Bin 1078 -> 1132 bytes
 85 files changed, 17394 insertions(+), 1976 deletions(-)
 create mode 100644 assets/javascripts/bundle.c8d2eff1.min.js
 create mode 100644 assets/javascripts/bundle.c8d2eff1.min.js.map
 delete mode 100644 assets/javascripts/bundle.febc23d1.min.js
 delete mode 100644 assets/javascripts/bundle.febc23d1.min.js.map
 create mode 100644 assets/javascripts/lunr/min/lunr.el.min.js
 create mode 100644 assets/javascripts/lunr/min/lunr.he.min.js
 create mode 100644 assets/javascripts/lunr/min/lunr.hy.min.js
 create mode 100644 assets/javascripts/lunr/min/lunr.kn.min.js
 create mode 100644 assets/javascripts/lunr/min/lunr.ko.min.js
 create mode 100644 assets/javascripts/lunr/min/lunr.sa.min.js
 create mode 100644 assets/javascripts/lunr/min/lunr.ta.min.js
 create mode 100644 assets/javascripts/lunr/min/lunr.te.min.js
 create mode 100644 assets/javascripts/workers/search.b8dbb3d2.min.js
 create mode 100644 assets/javascripts/workers/search.b8dbb3d2.min.js.map
 create mode 100644 assets/stylesheets/main.7e359304.min.css
 create mode 100644 assets/stylesheets/main.7e359304.min.css.map
 create mode 100644 assets/stylesheets/palette.06af60db.min.css
 create mode 100644 assets/stylesheets/palette.06af60db.min.css.map
 create mode 100644 language_model/containers/hpc_containers_apptainer/index.html
 create mode 100644 language_model/intro/index.html
 create mode 100644 language_model/llm/llm_intro_hyperparameter_tuning/index.html

diff --git a/404.html b/404.html
index b244e1961..6d02c5f6a 100644
--- a/404.html
+++ b/404.html
@@ -14,7 +14,7 @@
       
       
       <link rel="icon" href="/./docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -179,7 +179,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -254,6 +254,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="/language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="/machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -348,7 +364,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -907,11 +923,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="/language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="/language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="/language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -939,14 +1079,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -999,14 +1139,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1137,14 +1277,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1160,14 +1300,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1220,14 +1360,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1428,7 +1568,7 @@ <h1>404 - Not found</h1>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1491,12 +1631,16 @@ <h1>404 - Not found</h1>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "/", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "/assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "/", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "/assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="/assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="/javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/about/index.html b/about/index.html
index c7c6fe9ad..446592192 100644
--- a/about/index.html
+++ b/about/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/about/">
       
       <link rel="icon" href="../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -972,11 +988,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1004,14 +1144,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1064,14 +1204,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1202,14 +1342,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1225,14 +1365,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1285,14 +1425,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1538,7 +1678,7 @@ <h2 id="experienced-research-and-applied-core-team">Experienced Research and App
 </div>
 <div class="admonition abstract">
 <p class="admonition-title">Jie Fu</p>
-<p>Jie Fu (付杰) is a happy and funny machine learning researcher (principal investigator), at Beijing Academy of Artificial Intelligence2 (BAAI, 北京智源人工智能研究院), with his human-friendly big AI dream.</p>
+<p>Jie Fu is a happy and funny machine learning researcher (principal investigator), at Hong Kong University of Science and Technology (HKUST), with his human-friendly big AI dream.</p>
 <p>He worked as a postdoc with Yoshua Bengio at University of Montreal, Quebec AI Institute (Mila), funded by Microsoft Research Montreal. He was an IVADO postdoc fellow working with Chris Pal at Polytechnique Montreal, Quebec AI Institute (Mila). He obtained his PhD from National University of Singapore under the supervision of Tat-Seng Chua. He received ICLR 2021 Outstanding Paper Award.</p>
 <p>He is currently working towards system-2 deep learning and its adaption to various real-world tasks, including AI for Science (drug discovery in particular) and AI for Social Good that can benefit all of society. He is also broadly interested in general deep learning, reinforcement learning, and language processing.</p>
 <p>Check out my profile link at <a href="http://bigaidream.github.io/">bigaidream.github.io</a></p>
@@ -1634,7 +1774,7 @@ <h2 id="citation">Citation<a class="headerlink" href="#citation" title="Permanen
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1697,12 +1837,16 @@ <h2 id="citation">Citation<a class="headerlink" href="#citation" title="Permanen
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/assets/javascripts/bundle.c8d2eff1.min.js b/assets/javascripts/bundle.c8d2eff1.min.js
new file mode 100644
index 000000000..4b1b31f52
--- /dev/null
+++ b/assets/javascripts/bundle.c8d2eff1.min.js
@@ -0,0 +1,29 @@
+"use strict";(()=>{var _i=Object.create;var br=Object.defineProperty;var Ai=Object.getOwnPropertyDescriptor;var Ci=Object.getOwnPropertyNames,Ft=Object.getOwnPropertySymbols,ki=Object.getPrototypeOf,vr=Object.prototype.hasOwnProperty,eo=Object.prototype.propertyIsEnumerable;var Zr=(e,t,r)=>t in e?br(e,t,{enumerable:!0,configurable:!0,writable:!0,value:r}):e[t]=r,F=(e,t)=>{for(var r in t||(t={}))vr.call(t,r)&&Zr(e,r,t[r]);if(Ft)for(var r of Ft(t))eo.call(t,r)&&Zr(e,r,t[r]);return e};var to=(e,t)=>{var r={};for(var o in e)vr.call(e,o)&&t.indexOf(o)<0&&(r[o]=e[o]);if(e!=null&&Ft)for(var o of Ft(e))t.indexOf(o)<0&&eo.call(e,o)&&(r[o]=e[o]);return r};var gr=(e,t)=>()=>(t||e((t={exports:{}}).exports,t),t.exports);var Hi=(e,t,r,o)=>{if(t&&typeof t=="object"||typeof t=="function")for(let n of Ci(t))!vr.call(e,n)&&n!==r&&br(e,n,{get:()=>t[n],enumerable:!(o=Ai(t,n))||o.enumerable});return e};var jt=(e,t,r)=>(r=e!=null?_i(ki(e)):{},Hi(t||!e||!e.__esModule?br(r,"default",{value:e,enumerable:!0}):r,e));var ro=(e,t,r)=>new Promise((o,n)=>{var i=c=>{try{a(r.next(c))}catch(p){n(p)}},s=c=>{try{a(r.throw(c))}catch(p){n(p)}},a=c=>c.done?o(c.value):Promise.resolve(c.value).then(i,s);a((r=r.apply(e,t)).next())});var no=gr((xr,oo)=>{(function(e,t){typeof xr=="object"&&typeof oo!="undefined"?t():typeof define=="function"&&define.amd?define(t):t()})(xr,function(){"use strict";function e(r){var o=!0,n=!1,i=null,s={text:!0,search:!0,url:!0,tel:!0,email:!0,password:!0,number:!0,date:!0,month:!0,week:!0,time:!0,datetime:!0,"datetime-local":!0};function a(C){return!!(C&&C!==document&&C.nodeName!=="HTML"&&C.nodeName!=="BODY"&&"classList"in C&&"contains"in C.classList)}function c(C){var ct=C.type,Ne=C.tagName;return!!(Ne==="INPUT"&&s[ct]&&!C.readOnly||Ne==="TEXTAREA"&&!C.readOnly||C.isContentEditable)}function p(C){C.classList.contains("focus-visible")||(C.classList.add("focus-visible"),C.setAttribute("data-focus-visible-added",""))}function l(C){C.hasAttribute("data-focus-visible-added")&&(C.classList.remove("focus-visible"),C.removeAttribute("data-focus-visible-added"))}function f(C){C.metaKey||C.altKey||C.ctrlKey||(a(r.activeElement)&&p(r.activeElement),o=!0)}function u(C){o=!1}function h(C){a(C.target)&&(o||c(C.target))&&p(C.target)}function w(C){a(C.target)&&(C.target.classList.contains("focus-visible")||C.target.hasAttribute("data-focus-visible-added"))&&(n=!0,window.clearTimeout(i),i=window.setTimeout(function(){n=!1},100),l(C.target))}function A(C){document.visibilityState==="hidden"&&(n&&(o=!0),Z())}function Z(){document.addEventListener("mousemove",J),document.addEventListener("mousedown",J),document.addEventListener("mouseup",J),document.addEventListener("pointermove",J),document.addEventListener("pointerdown",J),document.addEventListener("pointerup",J),document.addEventListener("touchmove",J),document.addEventListener("touchstart",J),document.addEventListener("touchend",J)}function te(){document.removeEventListener("mousemove",J),document.removeEventListener("mousedown",J),document.removeEventListener("mouseup",J),document.removeEventListener("pointermove",J),document.removeEventListener("pointerdown",J),document.removeEventListener("pointerup",J),document.removeEventListener("touchmove",J),document.removeEventListener("touchstart",J),document.removeEventListener("touchend",J)}function J(C){C.target.nodeName&&C.target.nodeName.toLowerCase()==="html"||(o=!1,te())}document.addEventListener("keydown",f,!0),document.addEventListener("mousedown",u,!0),document.addEventListener("pointerdown",u,!0),document.addEventListener("touchstart",u,!0),document.addEventListener("visibilitychange",A,!0),Z(),r.addEventListener("focus",h,!0),r.addEventListener("blur",w,!0),r.nodeType===Node.DOCUMENT_FRAGMENT_NODE&&r.host?r.host.setAttribute("data-js-focus-visible",""):r.nodeType===Node.DOCUMENT_NODE&&(document.documentElement.classList.add("js-focus-visible"),document.documentElement.setAttribute("data-js-focus-visible",""))}if(typeof window!="undefined"&&typeof document!="undefined"){window.applyFocusVisiblePolyfill=e;var t;try{t=new CustomEvent("focus-visible-polyfill-ready")}catch(r){t=document.createEvent("CustomEvent"),t.initCustomEvent("focus-visible-polyfill-ready",!1,!1,{})}window.dispatchEvent(t)}typeof document!="undefined"&&e(document)})});var zr=gr((kt,Vr)=>{/*!
+ * clipboard.js v2.0.11
+ * https://clipboardjs.com/
+ *
+ * Licensed MIT © Zeno Rocha
+ */(function(t,r){typeof kt=="object"&&typeof Vr=="object"?Vr.exports=r():typeof define=="function"&&define.amd?define([],r):typeof kt=="object"?kt.ClipboardJS=r():t.ClipboardJS=r()})(kt,function(){return function(){var e={686:function(o,n,i){"use strict";i.d(n,{default:function(){return Li}});var s=i(279),a=i.n(s),c=i(370),p=i.n(c),l=i(817),f=i.n(l);function u(D){try{return document.execCommand(D)}catch(M){return!1}}var h=function(M){var O=f()(M);return u("cut"),O},w=h;function A(D){var M=document.documentElement.getAttribute("dir")==="rtl",O=document.createElement("textarea");O.style.fontSize="12pt",O.style.border="0",O.style.padding="0",O.style.margin="0",O.style.position="absolute",O.style[M?"right":"left"]="-9999px";var I=window.pageYOffset||document.documentElement.scrollTop;return O.style.top="".concat(I,"px"),O.setAttribute("readonly",""),O.value=D,O}var Z=function(M,O){var I=A(M);O.container.appendChild(I);var W=f()(I);return u("copy"),I.remove(),W},te=function(M){var O=arguments.length>1&&arguments[1]!==void 0?arguments[1]:{container:document.body},I="";return typeof M=="string"?I=Z(M,O):M instanceof HTMLInputElement&&!["text","search","url","tel","password"].includes(M==null?void 0:M.type)?I=Z(M.value,O):(I=f()(M),u("copy")),I},J=te;function C(D){"@babel/helpers - typeof";return typeof Symbol=="function"&&typeof Symbol.iterator=="symbol"?C=function(O){return typeof O}:C=function(O){return O&&typeof Symbol=="function"&&O.constructor===Symbol&&O!==Symbol.prototype?"symbol":typeof O},C(D)}var ct=function(){var M=arguments.length>0&&arguments[0]!==void 0?arguments[0]:{},O=M.action,I=O===void 0?"copy":O,W=M.container,K=M.target,Ce=M.text;if(I!=="copy"&&I!=="cut")throw new Error('Invalid "action" value, use either "copy" or "cut"');if(K!==void 0)if(K&&C(K)==="object"&&K.nodeType===1){if(I==="copy"&&K.hasAttribute("disabled"))throw new Error('Invalid "target" attribute. Please use "readonly" instead of "disabled" attribute');if(I==="cut"&&(K.hasAttribute("readonly")||K.hasAttribute("disabled")))throw new Error(`Invalid "target" attribute. You can't cut text from elements with "readonly" or "disabled" attributes`)}else throw new Error('Invalid "target" value, use a valid Element');if(Ce)return J(Ce,{container:W});if(K)return I==="cut"?w(K):J(K,{container:W})},Ne=ct;function Pe(D){"@babel/helpers - typeof";return typeof Symbol=="function"&&typeof Symbol.iterator=="symbol"?Pe=function(O){return typeof O}:Pe=function(O){return O&&typeof Symbol=="function"&&O.constructor===Symbol&&O!==Symbol.prototype?"symbol":typeof O},Pe(D)}function xi(D,M){if(!(D instanceof M))throw new TypeError("Cannot call a class as a function")}function Xr(D,M){for(var O=0;O<M.length;O++){var I=M[O];I.enumerable=I.enumerable||!1,I.configurable=!0,"value"in I&&(I.writable=!0),Object.defineProperty(D,I.key,I)}}function yi(D,M,O){return M&&Xr(D.prototype,M),O&&Xr(D,O),D}function Ei(D,M){if(typeof M!="function"&&M!==null)throw new TypeError("Super expression must either be null or a function");D.prototype=Object.create(M&&M.prototype,{constructor:{value:D,writable:!0,configurable:!0}}),M&&dr(D,M)}function dr(D,M){return dr=Object.setPrototypeOf||function(I,W){return I.__proto__=W,I},dr(D,M)}function wi(D){var M=Oi();return function(){var I=Pt(D),W;if(M){var K=Pt(this).constructor;W=Reflect.construct(I,arguments,K)}else W=I.apply(this,arguments);return Ti(this,W)}}function Ti(D,M){return M&&(Pe(M)==="object"||typeof M=="function")?M:Si(D)}function Si(D){if(D===void 0)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return D}function Oi(){if(typeof Reflect=="undefined"||!Reflect.construct||Reflect.construct.sham)return!1;if(typeof Proxy=="function")return!0;try{return Date.prototype.toString.call(Reflect.construct(Date,[],function(){})),!0}catch(D){return!1}}function Pt(D){return Pt=Object.setPrototypeOf?Object.getPrototypeOf:function(O){return O.__proto__||Object.getPrototypeOf(O)},Pt(D)}function hr(D,M){var O="data-clipboard-".concat(D);if(M.hasAttribute(O))return M.getAttribute(O)}var Mi=function(D){Ei(O,D);var M=wi(O);function O(I,W){var K;return xi(this,O),K=M.call(this),K.resolveOptions(W),K.listenClick(I),K}return yi(O,[{key:"resolveOptions",value:function(){var W=arguments.length>0&&arguments[0]!==void 0?arguments[0]:{};this.action=typeof W.action=="function"?W.action:this.defaultAction,this.target=typeof W.target=="function"?W.target:this.defaultTarget,this.text=typeof W.text=="function"?W.text:this.defaultText,this.container=Pe(W.container)==="object"?W.container:document.body}},{key:"listenClick",value:function(W){var K=this;this.listener=p()(W,"click",function(Ce){return K.onClick(Ce)})}},{key:"onClick",value:function(W){var K=W.delegateTarget||W.currentTarget,Ce=this.action(K)||"copy",It=Ne({action:Ce,container:this.container,target:this.target(K),text:this.text(K)});this.emit(It?"success":"error",{action:Ce,text:It,trigger:K,clearSelection:function(){K&&K.focus(),window.getSelection().removeAllRanges()}})}},{key:"defaultAction",value:function(W){return hr("action",W)}},{key:"defaultTarget",value:function(W){var K=hr("target",W);if(K)return document.querySelector(K)}},{key:"defaultText",value:function(W){return hr("text",W)}},{key:"destroy",value:function(){this.listener.destroy()}}],[{key:"copy",value:function(W){var K=arguments.length>1&&arguments[1]!==void 0?arguments[1]:{container:document.body};return J(W,K)}},{key:"cut",value:function(W){return w(W)}},{key:"isSupported",value:function(){var W=arguments.length>0&&arguments[0]!==void 0?arguments[0]:["copy","cut"],K=typeof W=="string"?[W]:W,Ce=!!document.queryCommandSupported;return K.forEach(function(It){Ce=Ce&&!!document.queryCommandSupported(It)}),Ce}}]),O}(a()),Li=Mi},828:function(o){var n=9;if(typeof Element!="undefined"&&!Element.prototype.matches){var i=Element.prototype;i.matches=i.matchesSelector||i.mozMatchesSelector||i.msMatchesSelector||i.oMatchesSelector||i.webkitMatchesSelector}function s(a,c){for(;a&&a.nodeType!==n;){if(typeof a.matches=="function"&&a.matches(c))return a;a=a.parentNode}}o.exports=s},438:function(o,n,i){var s=i(828);function a(l,f,u,h,w){var A=p.apply(this,arguments);return l.addEventListener(u,A,w),{destroy:function(){l.removeEventListener(u,A,w)}}}function c(l,f,u,h,w){return typeof l.addEventListener=="function"?a.apply(null,arguments):typeof u=="function"?a.bind(null,document).apply(null,arguments):(typeof l=="string"&&(l=document.querySelectorAll(l)),Array.prototype.map.call(l,function(A){return a(A,f,u,h,w)}))}function p(l,f,u,h){return function(w){w.delegateTarget=s(w.target,f),w.delegateTarget&&h.call(l,w)}}o.exports=c},879:function(o,n){n.node=function(i){return i!==void 0&&i instanceof HTMLElement&&i.nodeType===1},n.nodeList=function(i){var s=Object.prototype.toString.call(i);return i!==void 0&&(s==="[object NodeList]"||s==="[object HTMLCollection]")&&"length"in i&&(i.length===0||n.node(i[0]))},n.string=function(i){return typeof i=="string"||i instanceof String},n.fn=function(i){var s=Object.prototype.toString.call(i);return s==="[object Function]"}},370:function(o,n,i){var s=i(879),a=i(438);function c(u,h,w){if(!u&&!h&&!w)throw new Error("Missing required arguments");if(!s.string(h))throw new TypeError("Second argument must be a String");if(!s.fn(w))throw new TypeError("Third argument must be a Function");if(s.node(u))return p(u,h,w);if(s.nodeList(u))return l(u,h,w);if(s.string(u))return f(u,h,w);throw new TypeError("First argument must be a String, HTMLElement, HTMLCollection, or NodeList")}function p(u,h,w){return u.addEventListener(h,w),{destroy:function(){u.removeEventListener(h,w)}}}function l(u,h,w){return Array.prototype.forEach.call(u,function(A){A.addEventListener(h,w)}),{destroy:function(){Array.prototype.forEach.call(u,function(A){A.removeEventListener(h,w)})}}}function f(u,h,w){return a(document.body,u,h,w)}o.exports=c},817:function(o){function n(i){var s;if(i.nodeName==="SELECT")i.focus(),s=i.value;else if(i.nodeName==="INPUT"||i.nodeName==="TEXTAREA"){var a=i.hasAttribute("readonly");a||i.setAttribute("readonly",""),i.select(),i.setSelectionRange(0,i.value.length),a||i.removeAttribute("readonly"),s=i.value}else{i.hasAttribute("contenteditable")&&i.focus();var c=window.getSelection(),p=document.createRange();p.selectNodeContents(i),c.removeAllRanges(),c.addRange(p),s=c.toString()}return s}o.exports=n},279:function(o){function n(){}n.prototype={on:function(i,s,a){var c=this.e||(this.e={});return(c[i]||(c[i]=[])).push({fn:s,ctx:a}),this},once:function(i,s,a){var c=this;function p(){c.off(i,p),s.apply(a,arguments)}return p._=s,this.on(i,p,a)},emit:function(i){var s=[].slice.call(arguments,1),a=((this.e||(this.e={}))[i]||[]).slice(),c=0,p=a.length;for(c;c<p;c++)a[c].fn.apply(a[c].ctx,s);return this},off:function(i,s){var a=this.e||(this.e={}),c=a[i],p=[];if(c&&s)for(var l=0,f=c.length;l<f;l++)c[l].fn!==s&&c[l].fn._!==s&&p.push(c[l]);return p.length?a[i]=p:delete a[i],this}},o.exports=n,o.exports.TinyEmitter=n}},t={};function r(o){if(t[o])return t[o].exports;var n=t[o]={exports:{}};return e[o](n,n.exports,r),n.exports}return function(){r.n=function(o){var n=o&&o.__esModule?function(){return o.default}:function(){return o};return r.d(n,{a:n}),n}}(),function(){r.d=function(o,n){for(var i in n)r.o(n,i)&&!r.o(o,i)&&Object.defineProperty(o,i,{enumerable:!0,get:n[i]})}}(),function(){r.o=function(o,n){return Object.prototype.hasOwnProperty.call(o,n)}}(),r(686)}().default})});var Kn=gr((Cw,qn)=>{"use strict";/*!
+ * escape-html
+ * Copyright(c) 2012-2013 TJ Holowaychuk
+ * Copyright(c) 2015 Andreas Lubbe
+ * Copyright(c) 2015 Tiancheng "Timothy" Gu
+ * MIT Licensed
+ */var Va=/["'&<>]/;qn.exports=za;function za(e){var t=""+e,r=Va.exec(t);if(!r)return t;var o,n="",i=0,s=0;for(i=r.index;i<t.length;i++){switch(t.charCodeAt(i)){case 34:o="&quot;";break;case 38:o="&amp;";break;case 39:o="&#39;";break;case 60:o="&lt;";break;case 62:o="&gt;";break;default:continue}s!==i&&(n+=t.substring(s,i)),s=i+1,n+=o}return s!==i?n+t.substring(s,i):n}});var mM=jt(no());/*! *****************************************************************************
+Copyright (c) Microsoft Corporation.
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+***************************************************************************** */var yr=function(e,t){return yr=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(r,o){r.__proto__=o}||function(r,o){for(var n in o)Object.prototype.hasOwnProperty.call(o,n)&&(r[n]=o[n])},yr(e,t)};function se(e,t){if(typeof t!="function"&&t!==null)throw new TypeError("Class extends value "+String(t)+" is not a constructor or null");yr(e,t);function r(){this.constructor=e}e.prototype=t===null?Object.create(t):(r.prototype=t.prototype,new r)}function io(e,t,r,o){function n(i){return i instanceof r?i:new r(function(s){s(i)})}return new(r||(r=Promise))(function(i,s){function a(l){try{p(o.next(l))}catch(f){s(f)}}function c(l){try{p(o.throw(l))}catch(f){s(f)}}function p(l){l.done?i(l.value):n(l.value).then(a,c)}p((o=o.apply(e,t||[])).next())})}function Ut(e,t){var r={label:0,sent:function(){if(i[0]&1)throw i[1];return i[1]},trys:[],ops:[]},o,n,i,s;return s={next:a(0),throw:a(1),return:a(2)},typeof Symbol=="function"&&(s[Symbol.iterator]=function(){return this}),s;function a(p){return function(l){return c([p,l])}}function c(p){if(o)throw new TypeError("Generator is already executing.");for(;r;)try{if(o=1,n&&(i=p[0]&2?n.return:p[0]?n.throw||((i=n.return)&&i.call(n),0):n.next)&&!(i=i.call(n,p[1])).done)return i;switch(n=0,i&&(p=[p[0]&2,i.value]),p[0]){case 0:case 1:i=p;break;case 4:return r.label++,{value:p[1],done:!1};case 5:r.label++,n=p[1],p=[0];continue;case 7:p=r.ops.pop(),r.trys.pop();continue;default:if(i=r.trys,!(i=i.length>0&&i[i.length-1])&&(p[0]===6||p[0]===2)){r=0;continue}if(p[0]===3&&(!i||p[1]>i[0]&&p[1]<i[3])){r.label=p[1];break}if(p[0]===6&&r.label<i[1]){r.label=i[1],i=p;break}if(i&&r.label<i[2]){r.label=i[2],r.ops.push(p);break}i[2]&&r.ops.pop(),r.trys.pop();continue}p=t.call(e,r)}catch(l){p=[6,l],n=0}finally{o=i=0}if(p[0]&5)throw p[1];return{value:p[0]?p[1]:void 0,done:!0}}}function ue(e){var t=typeof Symbol=="function"&&Symbol.iterator,r=t&&e[t],o=0;if(r)return r.call(e);if(e&&typeof e.length=="number")return{next:function(){return e&&o>=e.length&&(e=void 0),{value:e&&e[o++],done:!e}}};throw new TypeError(t?"Object is not iterable.":"Symbol.iterator is not defined.")}function V(e,t){var r=typeof Symbol=="function"&&e[Symbol.iterator];if(!r)return e;var o=r.call(e),n,i=[],s;try{for(;(t===void 0||t-- >0)&&!(n=o.next()).done;)i.push(n.value)}catch(a){s={error:a}}finally{try{n&&!n.done&&(r=o.return)&&r.call(o)}finally{if(s)throw s.error}}return i}function z(e,t,r){if(r||arguments.length===2)for(var o=0,n=t.length,i;o<n;o++)(i||!(o in t))&&(i||(i=Array.prototype.slice.call(t,0,o)),i[o]=t[o]);return e.concat(i||Array.prototype.slice.call(t))}function ot(e){return this instanceof ot?(this.v=e,this):new ot(e)}function ao(e,t,r){if(!Symbol.asyncIterator)throw new TypeError("Symbol.asyncIterator is not defined.");var o=r.apply(e,t||[]),n,i=[];return n={},s("next"),s("throw"),s("return"),n[Symbol.asyncIterator]=function(){return this},n;function s(u){o[u]&&(n[u]=function(h){return new Promise(function(w,A){i.push([u,h,w,A])>1||a(u,h)})})}function a(u,h){try{c(o[u](h))}catch(w){f(i[0][3],w)}}function c(u){u.value instanceof ot?Promise.resolve(u.value.v).then(p,l):f(i[0][2],u)}function p(u){a("next",u)}function l(u){a("throw",u)}function f(u,h){u(h),i.shift(),i.length&&a(i[0][0],i[0][1])}}function so(e){if(!Symbol.asyncIterator)throw new TypeError("Symbol.asyncIterator is not defined.");var t=e[Symbol.asyncIterator],r;return t?t.call(e):(e=typeof ue=="function"?ue(e):e[Symbol.iterator](),r={},o("next"),o("throw"),o("return"),r[Symbol.asyncIterator]=function(){return this},r);function o(i){r[i]=e[i]&&function(s){return new Promise(function(a,c){s=e[i](s),n(a,c,s.done,s.value)})}}function n(i,s,a,c){Promise.resolve(c).then(function(p){i({value:p,done:a})},s)}}function k(e){return typeof e=="function"}function pt(e){var t=function(o){Error.call(o),o.stack=new Error().stack},r=e(t);return r.prototype=Object.create(Error.prototype),r.prototype.constructor=r,r}var Wt=pt(function(e){return function(r){e(this),this.message=r?r.length+` errors occurred during unsubscription:
+`+r.map(function(o,n){return n+1+") "+o.toString()}).join(`
+  `):"",this.name="UnsubscriptionError",this.errors=r}});function Ve(e,t){if(e){var r=e.indexOf(t);0<=r&&e.splice(r,1)}}var Ie=function(){function e(t){this.initialTeardown=t,this.closed=!1,this._parentage=null,this._finalizers=null}return e.prototype.unsubscribe=function(){var t,r,o,n,i;if(!this.closed){this.closed=!0;var s=this._parentage;if(s)if(this._parentage=null,Array.isArray(s))try{for(var a=ue(s),c=a.next();!c.done;c=a.next()){var p=c.value;p.remove(this)}}catch(A){t={error:A}}finally{try{c&&!c.done&&(r=a.return)&&r.call(a)}finally{if(t)throw t.error}}else s.remove(this);var l=this.initialTeardown;if(k(l))try{l()}catch(A){i=A instanceof Wt?A.errors:[A]}var f=this._finalizers;if(f){this._finalizers=null;try{for(var u=ue(f),h=u.next();!h.done;h=u.next()){var w=h.value;try{co(w)}catch(A){i=i!=null?i:[],A instanceof Wt?i=z(z([],V(i)),V(A.errors)):i.push(A)}}}catch(A){o={error:A}}finally{try{h&&!h.done&&(n=u.return)&&n.call(u)}finally{if(o)throw o.error}}}if(i)throw new Wt(i)}},e.prototype.add=function(t){var r;if(t&&t!==this)if(this.closed)co(t);else{if(t instanceof e){if(t.closed||t._hasParent(this))return;t._addParent(this)}(this._finalizers=(r=this._finalizers)!==null&&r!==void 0?r:[]).push(t)}},e.prototype._hasParent=function(t){var r=this._parentage;return r===t||Array.isArray(r)&&r.includes(t)},e.prototype._addParent=function(t){var r=this._parentage;this._parentage=Array.isArray(r)?(r.push(t),r):r?[r,t]:t},e.prototype._removeParent=function(t){var r=this._parentage;r===t?this._parentage=null:Array.isArray(r)&&Ve(r,t)},e.prototype.remove=function(t){var r=this._finalizers;r&&Ve(r,t),t instanceof e&&t._removeParent(this)},e.EMPTY=function(){var t=new e;return t.closed=!0,t}(),e}();var Er=Ie.EMPTY;function Dt(e){return e instanceof Ie||e&&"closed"in e&&k(e.remove)&&k(e.add)&&k(e.unsubscribe)}function co(e){k(e)?e():e.unsubscribe()}var ke={onUnhandledError:null,onStoppedNotification:null,Promise:void 0,useDeprecatedSynchronousErrorHandling:!1,useDeprecatedNextContext:!1};var lt={setTimeout:function(e,t){for(var r=[],o=2;o<arguments.length;o++)r[o-2]=arguments[o];var n=lt.delegate;return n!=null&&n.setTimeout?n.setTimeout.apply(n,z([e,t],V(r))):setTimeout.apply(void 0,z([e,t],V(r)))},clearTimeout:function(e){var t=lt.delegate;return((t==null?void 0:t.clearTimeout)||clearTimeout)(e)},delegate:void 0};function Nt(e){lt.setTimeout(function(){var t=ke.onUnhandledError;if(t)t(e);else throw e})}function Se(){}var po=function(){return wr("C",void 0,void 0)}();function lo(e){return wr("E",void 0,e)}function mo(e){return wr("N",e,void 0)}function wr(e,t,r){return{kind:e,value:t,error:r}}var nt=null;function mt(e){if(ke.useDeprecatedSynchronousErrorHandling){var t=!nt;if(t&&(nt={errorThrown:!1,error:null}),e(),t){var r=nt,o=r.errorThrown,n=r.error;if(nt=null,o)throw n}}else e()}function fo(e){ke.useDeprecatedSynchronousErrorHandling&&nt&&(nt.errorThrown=!0,nt.error=e)}var Tt=function(e){se(t,e);function t(r){var o=e.call(this)||this;return o.isStopped=!1,r?(o.destination=r,Dt(r)&&r.add(o)):o.destination=Ii,o}return t.create=function(r,o,n){return new it(r,o,n)},t.prototype.next=function(r){this.isStopped?Sr(mo(r),this):this._next(r)},t.prototype.error=function(r){this.isStopped?Sr(lo(r),this):(this.isStopped=!0,this._error(r))},t.prototype.complete=function(){this.isStopped?Sr(po,this):(this.isStopped=!0,this._complete())},t.prototype.unsubscribe=function(){this.closed||(this.isStopped=!0,e.prototype.unsubscribe.call(this),this.destination=null)},t.prototype._next=function(r){this.destination.next(r)},t.prototype._error=function(r){try{this.destination.error(r)}finally{this.unsubscribe()}},t.prototype._complete=function(){try{this.destination.complete()}finally{this.unsubscribe()}},t}(Ie);var $i=Function.prototype.bind;function Tr(e,t){return $i.call(e,t)}var Ri=function(){function e(t){this.partialObserver=t}return e.prototype.next=function(t){var r=this.partialObserver;if(r.next)try{r.next(t)}catch(o){Vt(o)}},e.prototype.error=function(t){var r=this.partialObserver;if(r.error)try{r.error(t)}catch(o){Vt(o)}else Vt(t)},e.prototype.complete=function(){var t=this.partialObserver;if(t.complete)try{t.complete()}catch(r){Vt(r)}},e}(),it=function(e){se(t,e);function t(r,o,n){var i=e.call(this)||this,s;if(k(r)||!r)s={next:r!=null?r:void 0,error:o!=null?o:void 0,complete:n!=null?n:void 0};else{var a;i&&ke.useDeprecatedNextContext?(a=Object.create(r),a.unsubscribe=function(){return i.unsubscribe()},s={next:r.next&&Tr(r.next,a),error:r.error&&Tr(r.error,a),complete:r.complete&&Tr(r.complete,a)}):s=r}return i.destination=new Ri(s),i}return t}(Tt);function Vt(e){ke.useDeprecatedSynchronousErrorHandling?fo(e):Nt(e)}function Pi(e){throw e}function Sr(e,t){var r=ke.onStoppedNotification;r&&lt.setTimeout(function(){return r(e,t)})}var Ii={closed:!0,next:Se,error:Pi,complete:Se};var ft=function(){return typeof Symbol=="function"&&Symbol.observable||"@@observable"}();function ce(e){return e}function uo(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];return Or(e)}function Or(e){return e.length===0?ce:e.length===1?e[0]:function(r){return e.reduce(function(o,n){return n(o)},r)}}var j=function(){function e(t){t&&(this._subscribe=t)}return e.prototype.lift=function(t){var r=new e;return r.source=this,r.operator=t,r},e.prototype.subscribe=function(t,r,o){var n=this,i=ji(t)?t:new it(t,r,o);return mt(function(){var s=n,a=s.operator,c=s.source;i.add(a?a.call(i,c):c?n._subscribe(i):n._trySubscribe(i))}),i},e.prototype._trySubscribe=function(t){try{return this._subscribe(t)}catch(r){t.error(r)}},e.prototype.forEach=function(t,r){var o=this;return r=ho(r),new r(function(n,i){var s=new it({next:function(a){try{t(a)}catch(c){i(c),s.unsubscribe()}},error:i,complete:n});o.subscribe(s)})},e.prototype._subscribe=function(t){var r;return(r=this.source)===null||r===void 0?void 0:r.subscribe(t)},e.prototype[ft]=function(){return this},e.prototype.pipe=function(){for(var t=[],r=0;r<arguments.length;r++)t[r]=arguments[r];return Or(t)(this)},e.prototype.toPromise=function(t){var r=this;return t=ho(t),new t(function(o,n){var i;r.subscribe(function(s){return i=s},function(s){return n(s)},function(){return o(i)})})},e.create=function(t){return new e(t)},e}();function ho(e){var t;return(t=e!=null?e:ke.Promise)!==null&&t!==void 0?t:Promise}function Fi(e){return e&&k(e.next)&&k(e.error)&&k(e.complete)}function ji(e){return e&&e instanceof Tt||Fi(e)&&Dt(e)}function Ui(e){return k(e==null?void 0:e.lift)}function x(e){return function(t){if(Ui(t))return t.lift(function(r){try{return e(r,this)}catch(o){this.error(o)}});throw new TypeError("Unable to lift unknown Observable type")}}function S(e,t,r,o,n){return new Wi(e,t,r,o,n)}var Wi=function(e){se(t,e);function t(r,o,n,i,s,a){var c=e.call(this,r)||this;return c.onFinalize=s,c.shouldUnsubscribe=a,c._next=o?function(p){try{o(p)}catch(l){r.error(l)}}:e.prototype._next,c._error=i?function(p){try{i(p)}catch(l){r.error(l)}finally{this.unsubscribe()}}:e.prototype._error,c._complete=n?function(){try{n()}catch(p){r.error(p)}finally{this.unsubscribe()}}:e.prototype._complete,c}return t.prototype.unsubscribe=function(){var r;if(!this.shouldUnsubscribe||this.shouldUnsubscribe()){var o=this.closed;e.prototype.unsubscribe.call(this),!o&&((r=this.onFinalize)===null||r===void 0||r.call(this))}},t}(Tt);var ut={schedule:function(e){var t=requestAnimationFrame,r=cancelAnimationFrame,o=ut.delegate;o&&(t=o.requestAnimationFrame,r=o.cancelAnimationFrame);var n=t(function(i){r=void 0,e(i)});return new Ie(function(){return r==null?void 0:r(n)})},requestAnimationFrame:function(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=ut.delegate;return((r==null?void 0:r.requestAnimationFrame)||requestAnimationFrame).apply(void 0,z([],V(e)))},cancelAnimationFrame:function(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=ut.delegate;return((r==null?void 0:r.cancelAnimationFrame)||cancelAnimationFrame).apply(void 0,z([],V(e)))},delegate:void 0};var bo=pt(function(e){return function(){e(this),this.name="ObjectUnsubscribedError",this.message="object unsubscribed"}});var v=function(e){se(t,e);function t(){var r=e.call(this)||this;return r.closed=!1,r.currentObservers=null,r.observers=[],r.isStopped=!1,r.hasError=!1,r.thrownError=null,r}return t.prototype.lift=function(r){var o=new vo(this,this);return o.operator=r,o},t.prototype._throwIfClosed=function(){if(this.closed)throw new bo},t.prototype.next=function(r){var o=this;mt(function(){var n,i;if(o._throwIfClosed(),!o.isStopped){o.currentObservers||(o.currentObservers=Array.from(o.observers));try{for(var s=ue(o.currentObservers),a=s.next();!a.done;a=s.next()){var c=a.value;c.next(r)}}catch(p){n={error:p}}finally{try{a&&!a.done&&(i=s.return)&&i.call(s)}finally{if(n)throw n.error}}}})},t.prototype.error=function(r){var o=this;mt(function(){if(o._throwIfClosed(),!o.isStopped){o.hasError=o.isStopped=!0,o.thrownError=r;for(var n=o.observers;n.length;)n.shift().error(r)}})},t.prototype.complete=function(){var r=this;mt(function(){if(r._throwIfClosed(),!r.isStopped){r.isStopped=!0;for(var o=r.observers;o.length;)o.shift().complete()}})},t.prototype.unsubscribe=function(){this.isStopped=this.closed=!0,this.observers=this.currentObservers=null},Object.defineProperty(t.prototype,"observed",{get:function(){var r;return((r=this.observers)===null||r===void 0?void 0:r.length)>0},enumerable:!1,configurable:!0}),t.prototype._trySubscribe=function(r){return this._throwIfClosed(),e.prototype._trySubscribe.call(this,r)},t.prototype._subscribe=function(r){return this._throwIfClosed(),this._checkFinalizedStatuses(r),this._innerSubscribe(r)},t.prototype._innerSubscribe=function(r){var o=this,n=this,i=n.hasError,s=n.isStopped,a=n.observers;return i||s?Er:(this.currentObservers=null,a.push(r),new Ie(function(){o.currentObservers=null,Ve(a,r)}))},t.prototype._checkFinalizedStatuses=function(r){var o=this,n=o.hasError,i=o.thrownError,s=o.isStopped;n?r.error(i):s&&r.complete()},t.prototype.asObservable=function(){var r=new j;return r.source=this,r},t.create=function(r,o){return new vo(r,o)},t}(j);var vo=function(e){se(t,e);function t(r,o){var n=e.call(this)||this;return n.destination=r,n.source=o,n}return t.prototype.next=function(r){var o,n;(n=(o=this.destination)===null||o===void 0?void 0:o.next)===null||n===void 0||n.call(o,r)},t.prototype.error=function(r){var o,n;(n=(o=this.destination)===null||o===void 0?void 0:o.error)===null||n===void 0||n.call(o,r)},t.prototype.complete=function(){var r,o;(o=(r=this.destination)===null||r===void 0?void 0:r.complete)===null||o===void 0||o.call(r)},t.prototype._subscribe=function(r){var o,n;return(n=(o=this.source)===null||o===void 0?void 0:o.subscribe(r))!==null&&n!==void 0?n:Er},t}(v);var St={now:function(){return(St.delegate||Date).now()},delegate:void 0};var Ot=function(e){se(t,e);function t(r,o,n){r===void 0&&(r=1/0),o===void 0&&(o=1/0),n===void 0&&(n=St);var i=e.call(this)||this;return i._bufferSize=r,i._windowTime=o,i._timestampProvider=n,i._buffer=[],i._infiniteTimeWindow=!0,i._infiniteTimeWindow=o===1/0,i._bufferSize=Math.max(1,r),i._windowTime=Math.max(1,o),i}return t.prototype.next=function(r){var o=this,n=o.isStopped,i=o._buffer,s=o._infiniteTimeWindow,a=o._timestampProvider,c=o._windowTime;n||(i.push(r),!s&&i.push(a.now()+c)),this._trimBuffer(),e.prototype.next.call(this,r)},t.prototype._subscribe=function(r){this._throwIfClosed(),this._trimBuffer();for(var o=this._innerSubscribe(r),n=this,i=n._infiniteTimeWindow,s=n._buffer,a=s.slice(),c=0;c<a.length&&!r.closed;c+=i?1:2)r.next(a[c]);return this._checkFinalizedStatuses(r),o},t.prototype._trimBuffer=function(){var r=this,o=r._bufferSize,n=r._timestampProvider,i=r._buffer,s=r._infiniteTimeWindow,a=(s?1:2)*o;if(o<1/0&&a<i.length&&i.splice(0,i.length-a),!s){for(var c=n.now(),p=0,l=1;l<i.length&&i[l]<=c;l+=2)p=l;p&&i.splice(0,p+1)}},t}(v);var go=function(e){se(t,e);function t(r,o){return e.call(this)||this}return t.prototype.schedule=function(r,o){return o===void 0&&(o=0),this},t}(Ie);var Mt={setInterval:function(e,t){for(var r=[],o=2;o<arguments.length;o++)r[o-2]=arguments[o];var n=Mt.delegate;return n!=null&&n.setInterval?n.setInterval.apply(n,z([e,t],V(r))):setInterval.apply(void 0,z([e,t],V(r)))},clearInterval:function(e){var t=Mt.delegate;return((t==null?void 0:t.clearInterval)||clearInterval)(e)},delegate:void 0};var zt=function(e){se(t,e);function t(r,o){var n=e.call(this,r,o)||this;return n.scheduler=r,n.work=o,n.pending=!1,n}return t.prototype.schedule=function(r,o){var n;if(o===void 0&&(o=0),this.closed)return this;this.state=r;var i=this.id,s=this.scheduler;return i!=null&&(this.id=this.recycleAsyncId(s,i,o)),this.pending=!0,this.delay=o,this.id=(n=this.id)!==null&&n!==void 0?n:this.requestAsyncId(s,this.id,o),this},t.prototype.requestAsyncId=function(r,o,n){return n===void 0&&(n=0),Mt.setInterval(r.flush.bind(r,this),n)},t.prototype.recycleAsyncId=function(r,o,n){if(n===void 0&&(n=0),n!=null&&this.delay===n&&this.pending===!1)return o;o!=null&&Mt.clearInterval(o)},t.prototype.execute=function(r,o){if(this.closed)return new Error("executing a cancelled action");this.pending=!1;var n=this._execute(r,o);if(n)return n;this.pending===!1&&this.id!=null&&(this.id=this.recycleAsyncId(this.scheduler,this.id,null))},t.prototype._execute=function(r,o){var n=!1,i;try{this.work(r)}catch(s){n=!0,i=s||new Error("Scheduled action threw falsy error")}if(n)return this.unsubscribe(),i},t.prototype.unsubscribe=function(){if(!this.closed){var r=this,o=r.id,n=r.scheduler,i=n.actions;this.work=this.state=this.scheduler=null,this.pending=!1,Ve(i,this),o!=null&&(this.id=this.recycleAsyncId(n,o,null)),this.delay=null,e.prototype.unsubscribe.call(this)}},t}(go);var Mr=function(){function e(t,r){r===void 0&&(r=e.now),this.schedulerActionCtor=t,this.now=r}return e.prototype.schedule=function(t,r,o){return r===void 0&&(r=0),new this.schedulerActionCtor(this,t).schedule(o,r)},e.now=St.now,e}();var qt=function(e){se(t,e);function t(r,o){o===void 0&&(o=Mr.now);var n=e.call(this,r,o)||this;return n.actions=[],n._active=!1,n}return t.prototype.flush=function(r){var o=this.actions;if(this._active){o.push(r);return}var n;this._active=!0;do if(n=r.execute(r.state,r.delay))break;while(r=o.shift());if(this._active=!1,n){for(;r=o.shift();)r.unsubscribe();throw n}},t}(Mr);var ie=new qt(zt),Lr=ie;var xo=function(e){se(t,e);function t(r,o){var n=e.call(this,r,o)||this;return n.scheduler=r,n.work=o,n}return t.prototype.requestAsyncId=function(r,o,n){return n===void 0&&(n=0),n!==null&&n>0?e.prototype.requestAsyncId.call(this,r,o,n):(r.actions.push(this),r._scheduled||(r._scheduled=ut.requestAnimationFrame(function(){return r.flush(void 0)})))},t.prototype.recycleAsyncId=function(r,o,n){var i;if(n===void 0&&(n=0),n!=null?n>0:this.delay>0)return e.prototype.recycleAsyncId.call(this,r,o,n);var s=r.actions;o!=null&&((i=s[s.length-1])===null||i===void 0?void 0:i.id)!==o&&(ut.cancelAnimationFrame(o),r._scheduled=void 0)},t}(zt);var yo=function(e){se(t,e);function t(){return e!==null&&e.apply(this,arguments)||this}return t.prototype.flush=function(r){this._active=!0;var o=this._scheduled;this._scheduled=void 0;var n=this.actions,i;r=r||n.shift();do if(i=r.execute(r.state,r.delay))break;while((r=n[0])&&r.id===o&&n.shift());if(this._active=!1,i){for(;(r=n[0])&&r.id===o&&n.shift();)r.unsubscribe();throw i}},t}(qt);var de=new yo(xo);var L=new j(function(e){return e.complete()});function Kt(e){return e&&k(e.schedule)}function _r(e){return e[e.length-1]}function Je(e){return k(_r(e))?e.pop():void 0}function Ae(e){return Kt(_r(e))?e.pop():void 0}function Qt(e,t){return typeof _r(e)=="number"?e.pop():t}var dt=function(e){return e&&typeof e.length=="number"&&typeof e!="function"};function Yt(e){return k(e==null?void 0:e.then)}function Bt(e){return k(e[ft])}function Gt(e){return Symbol.asyncIterator&&k(e==null?void 0:e[Symbol.asyncIterator])}function Jt(e){return new TypeError("You provided "+(e!==null&&typeof e=="object"?"an invalid object":"'"+e+"'")+" where a stream was expected. You can provide an Observable, Promise, ReadableStream, Array, AsyncIterable, or Iterable.")}function Di(){return typeof Symbol!="function"||!Symbol.iterator?"@@iterator":Symbol.iterator}var Xt=Di();function Zt(e){return k(e==null?void 0:e[Xt])}function er(e){return ao(this,arguments,function(){var r,o,n,i;return Ut(this,function(s){switch(s.label){case 0:r=e.getReader(),s.label=1;case 1:s.trys.push([1,,9,10]),s.label=2;case 2:return[4,ot(r.read())];case 3:return o=s.sent(),n=o.value,i=o.done,i?[4,ot(void 0)]:[3,5];case 4:return[2,s.sent()];case 5:return[4,ot(n)];case 6:return[4,s.sent()];case 7:return s.sent(),[3,2];case 8:return[3,10];case 9:return r.releaseLock(),[7];case 10:return[2]}})})}function tr(e){return k(e==null?void 0:e.getReader)}function N(e){if(e instanceof j)return e;if(e!=null){if(Bt(e))return Ni(e);if(dt(e))return Vi(e);if(Yt(e))return zi(e);if(Gt(e))return Eo(e);if(Zt(e))return qi(e);if(tr(e))return Ki(e)}throw Jt(e)}function Ni(e){return new j(function(t){var r=e[ft]();if(k(r.subscribe))return r.subscribe(t);throw new TypeError("Provided object does not correctly implement Symbol.observable")})}function Vi(e){return new j(function(t){for(var r=0;r<e.length&&!t.closed;r++)t.next(e[r]);t.complete()})}function zi(e){return new j(function(t){e.then(function(r){t.closed||(t.next(r),t.complete())},function(r){return t.error(r)}).then(null,Nt)})}function qi(e){return new j(function(t){var r,o;try{for(var n=ue(e),i=n.next();!i.done;i=n.next()){var s=i.value;if(t.next(s),t.closed)return}}catch(a){r={error:a}}finally{try{i&&!i.done&&(o=n.return)&&o.call(n)}finally{if(r)throw r.error}}t.complete()})}function Eo(e){return new j(function(t){Qi(e,t).catch(function(r){return t.error(r)})})}function Ki(e){return Eo(er(e))}function Qi(e,t){var r,o,n,i;return io(this,void 0,void 0,function(){var s,a;return Ut(this,function(c){switch(c.label){case 0:c.trys.push([0,5,6,11]),r=so(e),c.label=1;case 1:return[4,r.next()];case 2:if(o=c.sent(),!!o.done)return[3,4];if(s=o.value,t.next(s),t.closed)return[2];c.label=3;case 3:return[3,1];case 4:return[3,11];case 5:return a=c.sent(),n={error:a},[3,11];case 6:return c.trys.push([6,,9,10]),o&&!o.done&&(i=r.return)?[4,i.call(r)]:[3,8];case 7:c.sent(),c.label=8;case 8:return[3,10];case 9:if(n)throw n.error;return[7];case 10:return[7];case 11:return t.complete(),[2]}})})}function xe(e,t,r,o,n){o===void 0&&(o=0),n===void 0&&(n=!1);var i=t.schedule(function(){r(),n?e.add(this.schedule(null,o)):this.unsubscribe()},o);if(e.add(i),!n)return i}function Oe(e,t){return t===void 0&&(t=0),x(function(r,o){r.subscribe(S(o,function(n){return xe(o,e,function(){return o.next(n)},t)},function(){return xe(o,e,function(){return o.complete()},t)},function(n){return xe(o,e,function(){return o.error(n)},t)}))})}function ze(e,t){return t===void 0&&(t=0),x(function(r,o){o.add(e.schedule(function(){return r.subscribe(o)},t))})}function wo(e,t){return N(e).pipe(ze(t),Oe(t))}function To(e,t){return N(e).pipe(ze(t),Oe(t))}function So(e,t){return new j(function(r){var o=0;return t.schedule(function(){o===e.length?r.complete():(r.next(e[o++]),r.closed||this.schedule())})})}function Oo(e,t){return new j(function(r){var o;return xe(r,t,function(){o=e[Xt](),xe(r,t,function(){var n,i,s;try{n=o.next(),i=n.value,s=n.done}catch(a){r.error(a);return}s?r.complete():r.next(i)},0,!0)}),function(){return k(o==null?void 0:o.return)&&o.return()}})}function rr(e,t){if(!e)throw new Error("Iterable cannot be null");return new j(function(r){xe(r,t,function(){var o=e[Symbol.asyncIterator]();xe(r,t,function(){o.next().then(function(n){n.done?r.complete():r.next(n.value)})},0,!0)})})}function Mo(e,t){return rr(er(e),t)}function Lo(e,t){if(e!=null){if(Bt(e))return wo(e,t);if(dt(e))return So(e,t);if(Yt(e))return To(e,t);if(Gt(e))return rr(e,t);if(Zt(e))return Oo(e,t);if(tr(e))return Mo(e,t)}throw Jt(e)}function fe(e,t){return t?Lo(e,t):N(e)}function $(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=Ae(e);return fe(e,r)}function Ar(e,t){var r=k(e)?e:function(){return e},o=function(n){return n.error(r())};return new j(t?function(n){return t.schedule(o,0,n)}:o)}var or=pt(function(e){return function(){e(this),this.name="EmptyError",this.message="no elements in sequence"}});function _o(e){return e instanceof Date&&!isNaN(e)}function m(e,t){return x(function(r,o){var n=0;r.subscribe(S(o,function(i){o.next(e.call(t,i,n++))}))})}var Yi=Array.isArray;function Bi(e,t){return Yi(t)?e.apply(void 0,z([],V(t))):e(t)}function Xe(e){return m(function(t){return Bi(e,t)})}var Gi=Array.isArray,Ji=Object.getPrototypeOf,Xi=Object.prototype,Zi=Object.keys;function Ao(e){if(e.length===1){var t=e[0];if(Gi(t))return{args:t,keys:null};if(ea(t)){var r=Zi(t);return{args:r.map(function(o){return t[o]}),keys:r}}}return{args:e,keys:null}}function ea(e){return e&&typeof e=="object"&&Ji(e)===Xi}function Co(e,t){return e.reduce(function(r,o,n){return r[o]=t[n],r},{})}function Q(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=Ae(e),o=Je(e),n=Ao(e),i=n.args,s=n.keys;if(i.length===0)return fe([],r);var a=new j(Cr(i,r,s?function(c){return Co(s,c)}:ce));return o?a.pipe(Xe(o)):a}function Cr(e,t,r){return r===void 0&&(r=ce),function(o){ko(t,function(){for(var n=e.length,i=new Array(n),s=n,a=n,c=function(l){ko(t,function(){var f=fe(e[l],t),u=!1;f.subscribe(S(o,function(h){i[l]=h,u||(u=!0,a--),a||o.next(r(i.slice()))},function(){--s||o.complete()}))},o)},p=0;p<n;p++)c(p)},o)}}function ko(e,t,r){e?xe(r,e,t):t()}function Ho(e,t,r,o,n,i,s,a){var c=[],p=0,l=0,f=!1,u=function(){f&&!c.length&&!p&&t.complete()},h=function(A){return p<o?w(A):c.push(A)},w=function(A){i&&t.next(A),p++;var Z=!1;N(r(A,l++)).subscribe(S(t,function(te){n==null||n(te),i?h(te):t.next(te)},function(){Z=!0},void 0,function(){if(Z)try{p--;for(var te=function(){var J=c.shift();s?xe(t,s,function(){return w(J)}):w(J)};c.length&&p<o;)te();u()}catch(J){t.error(J)}}))};return e.subscribe(S(t,h,function(){f=!0,u()})),function(){a==null||a()}}function re(e,t,r){return r===void 0&&(r=1/0),k(t)?re(function(o,n){return m(function(i,s){return t(o,i,n,s)})(N(e(o,n)))},r):(typeof t=="number"&&(r=t),x(function(o,n){return Ho(o,n,e,r)}))}function ht(e){return e===void 0&&(e=1/0),re(ce,e)}function $o(){return ht(1)}function Fe(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];return $o()(fe(e,Ae(e)))}function H(e){return new j(function(t){N(e()).subscribe(t)})}var ta=["addListener","removeListener"],ra=["addEventListener","removeEventListener"],oa=["on","off"];function d(e,t,r,o){if(k(r)&&(o=r,r=void 0),o)return d(e,t,r).pipe(Xe(o));var n=V(aa(e)?ra.map(function(a){return function(c){return e[a](t,c,r)}}):na(e)?ta.map(Ro(e,t)):ia(e)?oa.map(Ro(e,t)):[],2),i=n[0],s=n[1];if(!i&&dt(e))return re(function(a){return d(a,t,r)})(N(e));if(!i)throw new TypeError("Invalid event target");return new j(function(a){var c=function(){for(var p=[],l=0;l<arguments.length;l++)p[l]=arguments[l];return a.next(1<p.length?p:p[0])};return i(c),function(){return s(c)}})}function Ro(e,t){return function(r){return function(o){return e[r](t,o)}}}function na(e){return k(e.addListener)&&k(e.removeListener)}function ia(e){return k(e.on)&&k(e.off)}function aa(e){return k(e.addEventListener)&&k(e.removeEventListener)}function nr(e,t,r){return r?nr(e,t).pipe(Xe(r)):new j(function(o){var n=function(){for(var s=[],a=0;a<arguments.length;a++)s[a]=arguments[a];return o.next(s.length===1?s[0]:s)},i=e(n);return k(t)?function(){return t(n,i)}:void 0})}function Ze(e,t,r){e===void 0&&(e=0),r===void 0&&(r=Lr);var o=-1;return t!=null&&(Kt(t)?r=t:o=t),new j(function(n){var i=_o(e)?+e-r.now():e;i<0&&(i=0);var s=0;return r.schedule(function(){n.closed||(n.next(s++),0<=o?this.schedule(void 0,o):n.complete())},i)})}function T(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=Ae(e),o=Qt(e,1/0),n=e;return n.length?n.length===1?N(n[0]):ht(o)(fe(n,r)):L}var qe=new j(Se);var sa=Array.isArray;function bt(e){return e.length===1&&sa(e[0])?e[0]:e}function g(e,t){return x(function(r,o){var n=0;r.subscribe(S(o,function(i){return e.call(t,i,n++)&&o.next(i)}))})}function Lt(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=Je(e),o=bt(e);return o.length?new j(function(n){var i=o.map(function(){return[]}),s=o.map(function(){return!1});n.add(function(){i=s=null});for(var a=function(p){N(o[p]).subscribe(S(n,function(l){if(i[p].push(l),i.every(function(u){return u.length})){var f=i.map(function(u){return u.shift()});n.next(r?r.apply(void 0,z([],V(f))):f),i.some(function(u,h){return!u.length&&s[h]})&&n.complete()}},function(){s[p]=!0,!i[p].length&&n.complete()}))},c=0;!n.closed&&c<o.length;c++)a(c);return function(){i=s=null}}):L}function Po(e){return x(function(t,r){var o=!1,n=null,i=null,s=!1,a=function(){if(i==null||i.unsubscribe(),i=null,o){o=!1;var p=n;n=null,r.next(p)}s&&r.complete()},c=function(){i=null,s&&r.complete()};t.subscribe(S(r,function(p){o=!0,n=p,i||N(e(p)).subscribe(i=S(r,a,c))},function(){s=!0,(!o||!i||i.closed)&&r.complete()}))})}function Me(e,t){return t===void 0&&(t=ie),Po(function(){return Ze(e,t)})}function Ke(e,t){return t===void 0&&(t=null),t=t!=null?t:e,x(function(r,o){var n=[],i=0;r.subscribe(S(o,function(s){var a,c,p,l,f=null;i++%t===0&&n.push([]);try{for(var u=ue(n),h=u.next();!h.done;h=u.next()){var w=h.value;w.push(s),e<=w.length&&(f=f!=null?f:[],f.push(w))}}catch(te){a={error:te}}finally{try{h&&!h.done&&(c=u.return)&&c.call(u)}finally{if(a)throw a.error}}if(f)try{for(var A=ue(f),Z=A.next();!Z.done;Z=A.next()){var w=Z.value;Ve(n,w),o.next(w)}}catch(te){p={error:te}}finally{try{Z&&!Z.done&&(l=A.return)&&l.call(A)}finally{if(p)throw p.error}}},function(){var s,a;try{for(var c=ue(n),p=c.next();!p.done;p=c.next()){var l=p.value;o.next(l)}}catch(f){s={error:f}}finally{try{p&&!p.done&&(a=c.return)&&a.call(c)}finally{if(s)throw s.error}}o.complete()},void 0,function(){n=null}))})}function he(e){return x(function(t,r){var o=null,n=!1,i;o=t.subscribe(S(r,void 0,void 0,function(s){i=N(e(s,he(e)(t))),o?(o.unsubscribe(),o=null,i.subscribe(r)):n=!0})),n&&(o.unsubscribe(),o=null,i.subscribe(r))})}function Io(e,t,r,o,n){return function(i,s){var a=r,c=t,p=0;i.subscribe(S(s,function(l){var f=p++;c=a?e(c,l,f):(a=!0,l),o&&s.next(c)},n&&function(){a&&s.next(c),s.complete()}))}}function kr(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=Je(e);return r?uo(kr.apply(void 0,z([],V(e))),Xe(r)):x(function(o,n){Cr(z([o],V(bt(e))))(n)})}function je(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];return kr.apply(void 0,z([],V(e)))}function be(e,t){return t===void 0&&(t=ie),x(function(r,o){var n=null,i=null,s=null,a=function(){if(n){n.unsubscribe(),n=null;var p=i;i=null,o.next(p)}};function c(){var p=s+e,l=t.now();if(l<p){n=this.schedule(void 0,p-l),o.add(n);return}a()}r.subscribe(S(o,function(p){i=p,s=t.now(),n||(n=t.schedule(c,e),o.add(n))},function(){a(),o.complete()},void 0,function(){i=n=null}))})}function Qe(e){return x(function(t,r){var o=!1;t.subscribe(S(r,function(n){o=!0,r.next(n)},function(){o||r.next(e),r.complete()}))})}function ye(e){return e<=0?function(){return L}:x(function(t,r){var o=0;t.subscribe(S(r,function(n){++o<=e&&(r.next(n),e<=o&&r.complete())}))})}function ee(){return x(function(e,t){e.subscribe(S(t,Se))})}function Fo(e){return m(function(){return e})}function Hr(e,t){return t?function(r){return Fe(t.pipe(ye(1),ee()),r.pipe(Hr(e)))}:re(function(r,o){return N(e(r,o)).pipe(ye(1),Fo(r))})}function Ye(e,t){t===void 0&&(t=ie);var r=Ze(e,t);return Hr(function(){return r})}function Y(e,t){return t===void 0&&(t=ce),e=e!=null?e:ca,x(function(r,o){var n,i=!0;r.subscribe(S(o,function(s){var a=t(s);(i||!e(n,a))&&(i=!1,n=a,o.next(s))}))})}function ca(e,t){return e===t}function X(e,t){return Y(function(r,o){return t?t(r[e],o[e]):r[e]===o[e]})}function jo(e){return e===void 0&&(e=pa),x(function(t,r){var o=!1;t.subscribe(S(r,function(n){o=!0,r.next(n)},function(){return o?r.complete():r.error(e())}))})}function pa(){return new or}function oe(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];return function(r){return Fe(r,$.apply(void 0,z([],V(e))))}}function _(e){return x(function(t,r){try{t.subscribe(r)}finally{r.add(e)}})}function He(e,t){var r=arguments.length>=2;return function(o){return o.pipe(e?g(function(n,i){return e(n,i,o)}):ce,ye(1),r?Qe(t):jo(function(){return new or}))}}function $r(e){return e<=0?function(){return L}:x(function(t,r){var o=[];t.subscribe(S(r,function(n){o.push(n),e<o.length&&o.shift()},function(){var n,i;try{for(var s=ue(o),a=s.next();!a.done;a=s.next()){var c=a.value;r.next(c)}}catch(p){n={error:p}}finally{try{a&&!a.done&&(i=s.return)&&i.call(s)}finally{if(n)throw n.error}}r.complete()},void 0,function(){o=null}))})}function Uo(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=Ae(e),o=Qt(e,1/0);return e=bt(e),x(function(n,i){ht(o)(fe(z([n],V(e)),r)).subscribe(i)})}function $e(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];return Uo.apply(void 0,z([],V(e)))}function at(e){var t,r=1/0,o;return e!=null&&(typeof e=="object"?(t=e.count,r=t===void 0?1/0:t,o=e.delay):r=e),r<=0?function(){return L}:x(function(n,i){var s=0,a,c=function(){if(a==null||a.unsubscribe(),a=null,o!=null){var l=typeof o=="number"?Ze(o):N(o(s)),f=S(i,function(){f.unsubscribe(),p()});l.subscribe(f)}else p()},p=function(){var l=!1;a=n.subscribe(S(i,void 0,function(){++s<r?a?c():l=!0:i.complete()})),l&&c()};p()})}function Rr(e,t){return x(Io(e,t,arguments.length>=2,!0))}function le(e){e===void 0&&(e={});var t=e.connector,r=t===void 0?function(){return new v}:t,o=e.resetOnError,n=o===void 0?!0:o,i=e.resetOnComplete,s=i===void 0?!0:i,a=e.resetOnRefCountZero,c=a===void 0?!0:a;return function(p){var l,f,u,h=0,w=!1,A=!1,Z=function(){f==null||f.unsubscribe(),f=void 0},te=function(){Z(),l=u=void 0,w=A=!1},J=function(){var C=l;te(),C==null||C.unsubscribe()};return x(function(C,ct){h++,!A&&!w&&Z();var Ne=u=u!=null?u:r();ct.add(function(){h--,h===0&&!A&&!w&&(f=Pr(J,c))}),Ne.subscribe(ct),!l&&h>0&&(l=new it({next:function(Pe){return Ne.next(Pe)},error:function(Pe){A=!0,Z(),f=Pr(te,n,Pe),Ne.error(Pe)},complete:function(){w=!0,Z(),f=Pr(te,s),Ne.complete()}}),N(C).subscribe(l))})(p)}}function Pr(e,t){for(var r=[],o=2;o<arguments.length;o++)r[o-2]=arguments[o];if(t===!0){e();return}if(t!==!1){var n=new it({next:function(){n.unsubscribe(),e()}});return N(t.apply(void 0,z([],V(r)))).subscribe(n)}}function B(e,t,r){var o,n,i,s,a=!1;return e&&typeof e=="object"?(o=e.bufferSize,s=o===void 0?1/0:o,n=e.windowTime,t=n===void 0?1/0:n,i=e.refCount,a=i===void 0?!1:i,r=e.scheduler):s=e!=null?e:1/0,le({connector:function(){return new Ot(s,t,r)},resetOnError:!0,resetOnComplete:!1,resetOnRefCountZero:a})}function Le(e){return g(function(t,r){return e<=r})}function Ir(e){return x(function(t,r){var o=!1,n=S(r,function(){n==null||n.unsubscribe(),o=!0},Se);N(e).subscribe(n),t.subscribe(S(r,function(i){return o&&r.next(i)}))})}function q(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=Ae(e);return x(function(o,n){(r?Fe(e,o,r):Fe(e,o)).subscribe(n)})}function b(e,t){return x(function(r,o){var n=null,i=0,s=!1,a=function(){return s&&!n&&o.complete()};r.subscribe(S(o,function(c){n==null||n.unsubscribe();var p=0,l=i++;N(e(c,l)).subscribe(n=S(o,function(f){return o.next(t?t(c,f,l,p++):f)},function(){n=null,a()}))},function(){s=!0,a()}))})}function U(e){return x(function(t,r){N(e).subscribe(S(r,function(){return r.complete()},Se)),!r.closed&&t.subscribe(r)})}function Fr(e,t){return t===void 0&&(t=!1),x(function(r,o){var n=0;r.subscribe(S(o,function(i){var s=e(i,n++);(s||t)&&o.next(i),!s&&o.complete()}))})}function y(e,t,r){var o=k(e)||t||r?{next:e,error:t,complete:r}:e;return o?x(function(n,i){var s;(s=o.subscribe)===null||s===void 0||s.call(o);var a=!0;n.subscribe(S(i,function(c){var p;(p=o.next)===null||p===void 0||p.call(o,c),i.next(c)},function(){var c;a=!1,(c=o.complete)===null||c===void 0||c.call(o),i.complete()},function(c){var p;a=!1,(p=o.error)===null||p===void 0||p.call(o,c),i.error(c)},function(){var c,p;a&&((c=o.unsubscribe)===null||c===void 0||c.call(o)),(p=o.finalize)===null||p===void 0||p.call(o)}))}):ce}function Wo(e,t){return x(function(r,o){var n=t!=null?t:{},i=n.leading,s=i===void 0?!0:i,a=n.trailing,c=a===void 0?!1:a,p=!1,l=null,f=null,u=!1,h=function(){f==null||f.unsubscribe(),f=null,c&&(Z(),u&&o.complete())},w=function(){f=null,u&&o.complete()},A=function(te){return f=N(e(te)).subscribe(S(o,h,w))},Z=function(){if(p){p=!1;var te=l;l=null,o.next(te),!u&&A(te)}};r.subscribe(S(o,function(te){p=!0,l=te,!(f&&!f.closed)&&(s?Z():A(te))},function(){u=!0,!(c&&p&&f&&!f.closed)&&o.complete()}))})}function _t(e,t,r){t===void 0&&(t=ie);var o=Ze(e,t);return Wo(function(){return o},r)}function ae(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=Je(e);return x(function(o,n){for(var i=e.length,s=new Array(i),a=e.map(function(){return!1}),c=!1,p=function(f){N(e[f]).subscribe(S(n,function(u){s[f]=u,!c&&!a[f]&&(a[f]=!0,(c=a.every(ce))&&(a=null))},Se))},l=0;l<i;l++)p(l);o.subscribe(S(n,function(f){if(c){var u=z([f],V(s));n.next(r?r.apply(void 0,z([],V(u))):u)}}))})}function Do(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];return x(function(r,o){Lt.apply(void 0,z([r],V(e))).subscribe(o)})}function jr(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];return Do.apply(void 0,z([],V(e)))}function No(){let e=new Ot(1);return d(document,"DOMContentLoaded",{once:!0}).subscribe(()=>e.next(document)),e}function R(e,t=document){return Array.from(t.querySelectorAll(e))}function P(e,t=document){let r=me(e,t);if(typeof r=="undefined")throw new ReferenceError(`Missing element: expected "${e}" to be present`);return r}function me(e,t=document){return t.querySelector(e)||void 0}function Re(){var e,t,r,o;return(o=(r=(t=(e=document.activeElement)==null?void 0:e.shadowRoot)==null?void 0:t.activeElement)!=null?r:document.activeElement)!=null?o:void 0}var la=T(d(document.body,"focusin"),d(document.body,"focusout")).pipe(be(1),q(void 0),m(()=>Re()||document.body),B(1));function vt(e){return la.pipe(m(t=>e.contains(t)),Y())}function Vo(e,t){return T(d(e,"mouseenter").pipe(m(()=>!0)),d(e,"mouseleave").pipe(m(()=>!1))).pipe(t?be(t):ce,q(!1))}function Ue(e){return{x:e.offsetLeft,y:e.offsetTop}}function zo(e){return T(d(window,"load"),d(window,"resize")).pipe(Me(0,de),m(()=>Ue(e)),q(Ue(e)))}function ir(e){return{x:e.scrollLeft,y:e.scrollTop}}function et(e){return T(d(e,"scroll"),d(window,"resize")).pipe(Me(0,de),m(()=>ir(e)),q(ir(e)))}function qo(e,t){if(typeof t=="string"||typeof t=="number")e.innerHTML+=t.toString();else if(t instanceof Node)e.appendChild(t);else if(Array.isArray(t))for(let r of t)qo(e,r)}function E(e,t,...r){let o=document.createElement(e);if(t)for(let n of Object.keys(t))typeof t[n]!="undefined"&&(typeof t[n]!="boolean"?o.setAttribute(n,t[n]):o.setAttribute(n,""));for(let n of r)qo(o,n);return o}function ar(e){if(e>999){let t=+((e-950)%1e3>99);return`${((e+1e-6)/1e3).toFixed(t)}k`}else return e.toString()}function gt(e){let t=E("script",{src:e});return H(()=>(document.head.appendChild(t),T(d(t,"load"),d(t,"error").pipe(b(()=>Ar(()=>new ReferenceError(`Invalid script: ${e}`))))).pipe(m(()=>{}),_(()=>document.head.removeChild(t)),ye(1))))}var Ko=new v,ma=H(()=>typeof ResizeObserver=="undefined"?gt("https://unpkg.com/resize-observer-polyfill"):$(void 0)).pipe(m(()=>new ResizeObserver(e=>{for(let t of e)Ko.next(t)})),b(e=>T(qe,$(e)).pipe(_(()=>e.disconnect()))),B(1));function pe(e){return{width:e.offsetWidth,height:e.offsetHeight}}function Ee(e){return ma.pipe(y(t=>t.observe(e)),b(t=>Ko.pipe(g(({target:r})=>r===e),_(()=>t.unobserve(e)),m(()=>pe(e)))),q(pe(e)))}function xt(e){return{width:e.scrollWidth,height:e.scrollHeight}}function sr(e){let t=e.parentElement;for(;t&&(e.scrollWidth<=t.scrollWidth&&e.scrollHeight<=t.scrollHeight);)t=(e=t).parentElement;return t?e:void 0}var Qo=new v,fa=H(()=>$(new IntersectionObserver(e=>{for(let t of e)Qo.next(t)},{threshold:0}))).pipe(b(e=>T(qe,$(e)).pipe(_(()=>e.disconnect()))),B(1));function yt(e){return fa.pipe(y(t=>t.observe(e)),b(t=>Qo.pipe(g(({target:r})=>r===e),_(()=>t.unobserve(e)),m(({isIntersecting:r})=>r))))}function Yo(e,t=16){return et(e).pipe(m(({y:r})=>{let o=pe(e),n=xt(e);return r>=n.height-o.height-t}),Y())}var cr={drawer:P("[data-md-toggle=drawer]"),search:P("[data-md-toggle=search]")};function Bo(e){return cr[e].checked}function Be(e,t){cr[e].checked!==t&&cr[e].click()}function We(e){let t=cr[e];return d(t,"change").pipe(m(()=>t.checked),q(t.checked))}function ua(e,t){switch(e.constructor){case HTMLInputElement:return e.type==="radio"?/^Arrow/.test(t):!0;case HTMLSelectElement:case HTMLTextAreaElement:return!0;default:return e.isContentEditable}}function da(){return T(d(window,"compositionstart").pipe(m(()=>!0)),d(window,"compositionend").pipe(m(()=>!1))).pipe(q(!1))}function Go(){let e=d(window,"keydown").pipe(g(t=>!(t.metaKey||t.ctrlKey)),m(t=>({mode:Bo("search")?"search":"global",type:t.key,claim(){t.preventDefault(),t.stopPropagation()}})),g(({mode:t,type:r})=>{if(t==="global"){let o=Re();if(typeof o!="undefined")return!ua(o,r)}return!0}),le());return da().pipe(b(t=>t?L:e))}function ve(){return new URL(location.href)}function st(e,t=!1){if(G("navigation.instant")&&!t){let r=E("a",{href:e.href});document.body.appendChild(r),r.click(),r.remove()}else location.href=e.href}function Jo(){return new v}function Xo(){return location.hash.slice(1)}function Zo(e){let t=E("a",{href:e});t.addEventListener("click",r=>r.stopPropagation()),t.click()}function ha(e){return T(d(window,"hashchange"),e).pipe(m(Xo),q(Xo()),g(t=>t.length>0),B(1))}function en(e){return ha(e).pipe(m(t=>me(`[id="${t}"]`)),g(t=>typeof t!="undefined"))}function At(e){let t=matchMedia(e);return nr(r=>t.addListener(()=>r(t.matches))).pipe(q(t.matches))}function tn(){let e=matchMedia("print");return T(d(window,"beforeprint").pipe(m(()=>!0)),d(window,"afterprint").pipe(m(()=>!1))).pipe(q(e.matches))}function Ur(e,t){return e.pipe(b(r=>r?t():L))}function Wr(e,t){return new j(r=>{let o=new XMLHttpRequest;return o.open("GET",`${e}`),o.responseType="blob",o.addEventListener("load",()=>{o.status>=200&&o.status<300?(r.next(o.response),r.complete()):r.error(new Error(o.statusText))}),o.addEventListener("error",()=>{r.error(new Error("Network error"))}),o.addEventListener("abort",()=>{r.complete()}),typeof(t==null?void 0:t.progress$)!="undefined"&&(o.addEventListener("progress",n=>{var i;if(n.lengthComputable)t.progress$.next(n.loaded/n.total*100);else{let s=(i=o.getResponseHeader("Content-Length"))!=null?i:0;t.progress$.next(n.loaded/+s*100)}}),t.progress$.next(5)),o.send(),()=>o.abort()})}function De(e,t){return Wr(e,t).pipe(b(r=>r.text()),m(r=>JSON.parse(r)),B(1))}function rn(e,t){let r=new DOMParser;return Wr(e,t).pipe(b(o=>o.text()),m(o=>r.parseFromString(o,"text/html")),B(1))}function on(e,t){let r=new DOMParser;return Wr(e,t).pipe(b(o=>o.text()),m(o=>r.parseFromString(o,"text/xml")),B(1))}function nn(){return{x:Math.max(0,scrollX),y:Math.max(0,scrollY)}}function an(){return T(d(window,"scroll",{passive:!0}),d(window,"resize",{passive:!0})).pipe(m(nn),q(nn()))}function sn(){return{width:innerWidth,height:innerHeight}}function cn(){return d(window,"resize",{passive:!0}).pipe(m(sn),q(sn()))}function pn(){return Q([an(),cn()]).pipe(m(([e,t])=>({offset:e,size:t})),B(1))}function pr(e,{viewport$:t,header$:r}){let o=t.pipe(X("size")),n=Q([o,r]).pipe(m(()=>Ue(e)));return Q([r,t,n]).pipe(m(([{height:i},{offset:s,size:a},{x:c,y:p}])=>({offset:{x:s.x-c,y:s.y-p+i},size:a})))}function ba(e){return d(e,"message",t=>t.data)}function va(e){let t=new v;return t.subscribe(r=>e.postMessage(r)),t}function ln(e,t=new Worker(e)){let r=ba(t),o=va(t),n=new v;n.subscribe(o);let i=o.pipe(ee(),oe(!0));return n.pipe(ee(),$e(r.pipe(U(i))),le())}var ga=P("#__config"),Et=JSON.parse(ga.textContent);Et.base=`${new URL(Et.base,ve())}`;function we(){return Et}function G(e){return Et.features.includes(e)}function ge(e,t){return typeof t!="undefined"?Et.translations[e].replace("#",t.toString()):Et.translations[e]}function Te(e,t=document){return P(`[data-md-component=${e}]`,t)}function ne(e,t=document){return R(`[data-md-component=${e}]`,t)}function xa(e){let t=P(".md-typeset > :first-child",e);return d(t,"click",{once:!0}).pipe(m(()=>P(".md-typeset",e)),m(r=>({hash:__md_hash(r.innerHTML)})))}function mn(e){if(!G("announce.dismiss")||!e.childElementCount)return L;if(!e.hidden){let t=P(".md-typeset",e);__md_hash(t.innerHTML)===__md_get("__announce")&&(e.hidden=!0)}return H(()=>{let t=new v;return t.subscribe(({hash:r})=>{e.hidden=!0,__md_set("__announce",r)}),xa(e).pipe(y(r=>t.next(r)),_(()=>t.complete()),m(r=>F({ref:e},r)))})}function ya(e,{target$:t}){return t.pipe(m(r=>({hidden:r!==e})))}function fn(e,t){let r=new v;return r.subscribe(({hidden:o})=>{e.hidden=o}),ya(e,t).pipe(y(o=>r.next(o)),_(()=>r.complete()),m(o=>F({ref:e},o)))}function Ct(e,t){return t==="inline"?E("div",{class:"md-tooltip md-tooltip--inline",id:e,role:"tooltip"},E("div",{class:"md-tooltip__inner md-typeset"})):E("div",{class:"md-tooltip",id:e,role:"tooltip"},E("div",{class:"md-tooltip__inner md-typeset"}))}function un(e,t){if(t=t?`${t}_annotation_${e}`:void 0,t){let r=t?`#${t}`:void 0;return E("aside",{class:"md-annotation",tabIndex:0},Ct(t),E("a",{href:r,class:"md-annotation__index",tabIndex:-1},E("span",{"data-md-annotation-id":e})))}else return E("aside",{class:"md-annotation",tabIndex:0},Ct(t),E("span",{class:"md-annotation__index",tabIndex:-1},E("span",{"data-md-annotation-id":e})))}function dn(e){return E("button",{class:"md-clipboard md-icon",title:ge("clipboard.copy"),"data-clipboard-target":`#${e} > code`})}function Dr(e,t){let r=t&2,o=t&1,n=Object.keys(e.terms).filter(c=>!e.terms[c]).reduce((c,p)=>[...c,E("del",null,p)," "],[]).slice(0,-1),i=we(),s=new URL(e.location,i.base);G("search.highlight")&&s.searchParams.set("h",Object.entries(e.terms).filter(([,c])=>c).reduce((c,[p])=>`${c} ${p}`.trim(),""));let{tags:a}=we();return E("a",{href:`${s}`,class:"md-search-result__link",tabIndex:-1},E("article",{class:"md-search-result__article md-typeset","data-md-score":e.score.toFixed(2)},r>0&&E("div",{class:"md-search-result__icon md-icon"}),r>0&&E("h1",null,e.title),r<=0&&E("h2",null,e.title),o>0&&e.text.length>0&&e.text,e.tags&&e.tags.map(c=>{let p=a?c in a?`md-tag-icon md-tag--${a[c]}`:"md-tag-icon":"";return E("span",{class:`md-tag ${p}`},c)}),o>0&&n.length>0&&E("p",{class:"md-search-result__terms"},ge("search.result.term.missing"),": ",...n)))}function hn(e){let t=e[0].score,r=[...e],o=we(),n=r.findIndex(l=>!`${new URL(l.location,o.base)}`.includes("#")),[i]=r.splice(n,1),s=r.findIndex(l=>l.score<t);s===-1&&(s=r.length);let a=r.slice(0,s),c=r.slice(s),p=[Dr(i,2|+(!n&&s===0)),...a.map(l=>Dr(l,1)),...c.length?[E("details",{class:"md-search-result__more"},E("summary",{tabIndex:-1},E("div",null,c.length>0&&c.length===1?ge("search.result.more.one"):ge("search.result.more.other",c.length))),...c.map(l=>Dr(l,1)))]:[]];return E("li",{class:"md-search-result__item"},p)}function bn(e){return E("ul",{class:"md-source__facts"},Object.entries(e).map(([t,r])=>E("li",{class:`md-source__fact md-source__fact--${t}`},typeof r=="number"?ar(r):r)))}function Nr(e){let t=`tabbed-control tabbed-control--${e}`;return E("div",{class:t,hidden:!0},E("button",{class:"tabbed-button",tabIndex:-1,"aria-hidden":"true"}))}function vn(e){return E("div",{class:"md-typeset__scrollwrap"},E("div",{class:"md-typeset__table"},e))}function Ea(e){let t=we(),r=new URL(`../${e.version}/`,t.base);return E("li",{class:"md-version__item"},E("a",{href:`${r}`,class:"md-version__link"},e.title))}function gn(e,t){return E("div",{class:"md-version"},E("button",{class:"md-version__current","aria-label":ge("select.version")},t.title),E("ul",{class:"md-version__list"},e.map(Ea)))}var wa=0;function Ta(e,t){document.body.append(e);let{width:r}=pe(e);e.style.setProperty("--md-tooltip-width",`${r}px`),e.remove();let o=sr(t),n=typeof o!="undefined"?et(o):$({x:0,y:0}),i=T(vt(t),Vo(t)).pipe(Y());return Q([i,n]).pipe(m(([s,a])=>{let{x:c,y:p}=Ue(t),l=pe(t),f=t.closest("table");return f&&t.parentElement&&(c+=f.offsetLeft+t.parentElement.offsetLeft,p+=f.offsetTop+t.parentElement.offsetTop),{active:s,offset:{x:c-a.x+l.width/2-r/2,y:p-a.y+l.height+8}}}))}function Ge(e){let t=e.title;if(!t.length)return L;let r=`__tooltip_${wa++}`,o=Ct(r,"inline"),n=P(".md-typeset",o);return n.innerHTML=t,H(()=>{let i=new v;return i.subscribe({next({offset:s}){o.style.setProperty("--md-tooltip-x",`${s.x}px`),o.style.setProperty("--md-tooltip-y",`${s.y}px`)},complete(){o.style.removeProperty("--md-tooltip-x"),o.style.removeProperty("--md-tooltip-y")}}),T(i.pipe(g(({active:s})=>s)),i.pipe(be(250),g(({active:s})=>!s))).subscribe({next({active:s}){s?(e.insertAdjacentElement("afterend",o),e.setAttribute("aria-describedby",r),e.removeAttribute("title")):(o.remove(),e.removeAttribute("aria-describedby"),e.setAttribute("title",t))},complete(){o.remove(),e.removeAttribute("aria-describedby"),e.setAttribute("title",t)}}),i.pipe(Me(16,de)).subscribe(({active:s})=>{o.classList.toggle("md-tooltip--active",s)}),i.pipe(_t(125,de),g(()=>!!e.offsetParent),m(()=>e.offsetParent.getBoundingClientRect()),m(({x:s})=>s)).subscribe({next(s){s?o.style.setProperty("--md-tooltip-0",`${-s}px`):o.style.removeProperty("--md-tooltip-0")},complete(){o.style.removeProperty("--md-tooltip-0")}}),Ta(o,e).pipe(y(s=>i.next(s)),_(()=>i.complete()),m(s=>F({ref:e},s)))}).pipe(ze(ie))}function Sa(e,t){let r=H(()=>Q([zo(e),et(t)])).pipe(m(([{x:o,y:n},i])=>{let{width:s,height:a}=pe(e);return{x:o-i.x+s/2,y:n-i.y+a/2}}));return vt(e).pipe(b(o=>r.pipe(m(n=>({active:o,offset:n})),ye(+!o||1/0))))}function xn(e,t,{target$:r}){let[o,n]=Array.from(e.children);return H(()=>{let i=new v,s=i.pipe(ee(),oe(!0));return i.subscribe({next({offset:a}){e.style.setProperty("--md-tooltip-x",`${a.x}px`),e.style.setProperty("--md-tooltip-y",`${a.y}px`)},complete(){e.style.removeProperty("--md-tooltip-x"),e.style.removeProperty("--md-tooltip-y")}}),yt(e).pipe(U(s)).subscribe(a=>{e.toggleAttribute("data-md-visible",a)}),T(i.pipe(g(({active:a})=>a)),i.pipe(be(250),g(({active:a})=>!a))).subscribe({next({active:a}){a?e.prepend(o):o.remove()},complete(){e.prepend(o)}}),i.pipe(Me(16,de)).subscribe(({active:a})=>{o.classList.toggle("md-tooltip--active",a)}),i.pipe(_t(125,de),g(()=>!!e.offsetParent),m(()=>e.offsetParent.getBoundingClientRect()),m(({x:a})=>a)).subscribe({next(a){a?e.style.setProperty("--md-tooltip-0",`${-a}px`):e.style.removeProperty("--md-tooltip-0")},complete(){e.style.removeProperty("--md-tooltip-0")}}),d(n,"click").pipe(U(s),g(a=>!(a.metaKey||a.ctrlKey))).subscribe(a=>{a.stopPropagation(),a.preventDefault()}),d(n,"mousedown").pipe(U(s),ae(i)).subscribe(([a,{active:c}])=>{var p;if(a.button!==0||a.metaKey||a.ctrlKey)a.preventDefault();else if(c){a.preventDefault();let l=e.parentElement.closest(".md-annotation");l instanceof HTMLElement?l.focus():(p=Re())==null||p.blur()}}),r.pipe(U(s),g(a=>a===o),Ye(125)).subscribe(()=>e.focus()),Sa(e,t).pipe(y(a=>i.next(a)),_(()=>i.complete()),m(a=>F({ref:e},a)))})}function Oa(e){return e.tagName==="CODE"?R(".c, .c1, .cm",e):[e]}function Ma(e){let t=[];for(let r of Oa(e)){let o=[],n=document.createNodeIterator(r,NodeFilter.SHOW_TEXT);for(let i=n.nextNode();i;i=n.nextNode())o.push(i);for(let i of o){let s;for(;s=/(\(\d+\))(!)?/.exec(i.textContent);){let[,a,c]=s;if(typeof c=="undefined"){let p=i.splitText(s.index);i=p.splitText(a.length),t.push(p)}else{i.textContent=a,t.push(i);break}}}}return t}function yn(e,t){t.append(...Array.from(e.childNodes))}function lr(e,t,{target$:r,print$:o}){let n=t.closest("[id]"),i=n==null?void 0:n.id,s=new Map;for(let a of Ma(t)){let[,c]=a.textContent.match(/\((\d+)\)/);me(`:scope > li:nth-child(${c})`,e)&&(s.set(c,un(c,i)),a.replaceWith(s.get(c)))}return s.size===0?L:H(()=>{let a=new v,c=a.pipe(ee(),oe(!0)),p=[];for(let[l,f]of s)p.push([P(".md-typeset",f),P(`:scope > li:nth-child(${l})`,e)]);return o.pipe(U(c)).subscribe(l=>{e.hidden=!l,e.classList.toggle("md-annotation-list",l);for(let[f,u]of p)l?yn(f,u):yn(u,f)}),T(...[...s].map(([,l])=>xn(l,t,{target$:r}))).pipe(_(()=>a.complete()),le())})}function En(e){if(e.nextElementSibling){let t=e.nextElementSibling;if(t.tagName==="OL")return t;if(t.tagName==="P"&&!t.children.length)return En(t)}}function wn(e,t){return H(()=>{let r=En(e);return typeof r!="undefined"?lr(r,e,t):L})}var Tn=jt(zr());var La=0;function Sn(e){if(e.nextElementSibling){let t=e.nextElementSibling;if(t.tagName==="OL")return t;if(t.tagName==="P"&&!t.children.length)return Sn(t)}}function _a(e){return Ee(e).pipe(m(({width:t})=>({scrollable:xt(e).width>t})),X("scrollable"))}function On(e,t){let{matches:r}=matchMedia("(hover)"),o=H(()=>{let n=new v,i=n.pipe($r(1));n.subscribe(({scrollable:c})=>{c&&r?e.setAttribute("tabindex","0"):e.removeAttribute("tabindex")});let s=[];if(Tn.default.isSupported()&&(e.closest(".copy")||G("content.code.copy")&&!e.closest(".no-copy"))){let c=e.closest("pre");c.id=`__code_${La++}`;let p=dn(c.id);c.insertBefore(p,e),G("content.tooltips")&&s.push(Ge(p))}let a=e.closest(".highlight");if(a instanceof HTMLElement){let c=Sn(a);if(typeof c!="undefined"&&(a.classList.contains("annotate")||G("content.code.annotate"))){let p=lr(c,e,t);s.push(Ee(a).pipe(U(i),m(({width:l,height:f})=>l&&f),Y(),b(l=>l?p:L)))}}return _a(e).pipe(y(c=>n.next(c)),_(()=>n.complete()),m(c=>F({ref:e},c)),$e(...s))});return G("content.lazy")?yt(e).pipe(g(n=>n),ye(1),b(()=>o)):o}function Aa(e,{target$:t,print$:r}){let o=!0;return T(t.pipe(m(n=>n.closest("details:not([open])")),g(n=>e===n),m(()=>({action:"open",reveal:!0}))),r.pipe(g(n=>n||!o),y(()=>o=e.open),m(n=>({action:n?"open":"close"}))))}function Mn(e,t){return H(()=>{let r=new v;return r.subscribe(({action:o,reveal:n})=>{e.toggleAttribute("open",o==="open"),n&&e.scrollIntoView()}),Aa(e,t).pipe(y(o=>r.next(o)),_(()=>r.complete()),m(o=>F({ref:e},o)))})}var Ln=".node circle,.node ellipse,.node path,.node polygon,.node rect{fill:var(--md-mermaid-node-bg-color);stroke:var(--md-mermaid-node-fg-color)}marker{fill:var(--md-mermaid-edge-color)!important}.edgeLabel .label rect{fill:#0000}.label{color:var(--md-mermaid-label-fg-color);font-family:var(--md-mermaid-font-family)}.label foreignObject{line-height:normal;overflow:visible}.label div .edgeLabel{color:var(--md-mermaid-label-fg-color)}.edgeLabel,.edgeLabel rect,.label div .edgeLabel{background-color:var(--md-mermaid-label-bg-color)}.edgeLabel,.edgeLabel rect{fill:var(--md-mermaid-label-bg-color);color:var(--md-mermaid-edge-color)}.edgePath .path,.flowchart-link{stroke:var(--md-mermaid-edge-color);stroke-width:.05rem}.edgePath .arrowheadPath{fill:var(--md-mermaid-edge-color);stroke:none}.cluster rect{fill:var(--md-default-fg-color--lightest);stroke:var(--md-default-fg-color--lighter)}.cluster span{color:var(--md-mermaid-label-fg-color);font-family:var(--md-mermaid-font-family)}g #flowchart-circleEnd,g #flowchart-circleStart,g #flowchart-crossEnd,g #flowchart-crossStart,g #flowchart-pointEnd,g #flowchart-pointStart{stroke:none}g.classGroup line,g.classGroup rect{fill:var(--md-mermaid-node-bg-color);stroke:var(--md-mermaid-node-fg-color)}g.classGroup text{fill:var(--md-mermaid-label-fg-color);font-family:var(--md-mermaid-font-family)}.classLabel .box{fill:var(--md-mermaid-label-bg-color);background-color:var(--md-mermaid-label-bg-color);opacity:1}.classLabel .label{fill:var(--md-mermaid-label-fg-color);font-family:var(--md-mermaid-font-family)}.node .divider{stroke:var(--md-mermaid-node-fg-color)}.relation{stroke:var(--md-mermaid-edge-color)}.cardinality{fill:var(--md-mermaid-label-fg-color);font-family:var(--md-mermaid-font-family)}.cardinality text{fill:inherit!important}defs #classDiagram-compositionEnd,defs #classDiagram-compositionStart,defs #classDiagram-dependencyEnd,defs #classDiagram-dependencyStart,defs #classDiagram-extensionEnd,defs #classDiagram-extensionStart{fill:var(--md-mermaid-edge-color)!important;stroke:var(--md-mermaid-edge-color)!important}defs #classDiagram-aggregationEnd,defs #classDiagram-aggregationStart{fill:var(--md-mermaid-label-bg-color)!important;stroke:var(--md-mermaid-edge-color)!important}g.stateGroup rect{fill:var(--md-mermaid-node-bg-color);stroke:var(--md-mermaid-node-fg-color)}g.stateGroup .state-title{fill:var(--md-mermaid-label-fg-color)!important;font-family:var(--md-mermaid-font-family)}g.stateGroup .composit{fill:var(--md-mermaid-label-bg-color)}.nodeLabel,.nodeLabel p{color:var(--md-mermaid-label-fg-color);font-family:var(--md-mermaid-font-family)}.node circle.state-end,.node circle.state-start,.start-state{fill:var(--md-mermaid-edge-color);stroke:none}.end-state-inner,.end-state-outer{fill:var(--md-mermaid-edge-color)}.end-state-inner,.node circle.state-end{stroke:var(--md-mermaid-label-bg-color)}.transition{stroke:var(--md-mermaid-edge-color)}[id^=state-fork] rect,[id^=state-join] rect{fill:var(--md-mermaid-edge-color)!important;stroke:none!important}.statediagram-cluster.statediagram-cluster .inner{fill:var(--md-default-bg-color)}.statediagram-cluster rect{fill:var(--md-mermaid-node-bg-color);stroke:var(--md-mermaid-node-fg-color)}.statediagram-state rect.divider{fill:var(--md-default-fg-color--lightest);stroke:var(--md-default-fg-color--lighter)}defs #statediagram-barbEnd{stroke:var(--md-mermaid-edge-color)}.attributeBoxEven,.attributeBoxOdd{fill:var(--md-mermaid-node-bg-color);stroke:var(--md-mermaid-node-fg-color)}.entityBox{fill:var(--md-mermaid-label-bg-color);stroke:var(--md-mermaid-node-fg-color)}.entityLabel{fill:var(--md-mermaid-label-fg-color);font-family:var(--md-mermaid-font-family)}.relationshipLabelBox{fill:var(--md-mermaid-label-bg-color);fill-opacity:1;background-color:var(--md-mermaid-label-bg-color);opacity:1}.relationshipLabel{fill:var(--md-mermaid-label-fg-color)}.relationshipLine{stroke:var(--md-mermaid-edge-color)}defs #ONE_OR_MORE_END *,defs #ONE_OR_MORE_START *,defs #ONLY_ONE_END *,defs #ONLY_ONE_START *,defs #ZERO_OR_MORE_END *,defs #ZERO_OR_MORE_START *,defs #ZERO_OR_ONE_END *,defs #ZERO_OR_ONE_START *{stroke:var(--md-mermaid-edge-color)!important}defs #ZERO_OR_MORE_END circle,defs #ZERO_OR_MORE_START circle{fill:var(--md-mermaid-label-bg-color)}.actor{fill:var(--md-mermaid-sequence-actor-bg-color);stroke:var(--md-mermaid-sequence-actor-border-color)}text.actor>tspan{fill:var(--md-mermaid-sequence-actor-fg-color);font-family:var(--md-mermaid-font-family)}line{stroke:var(--md-mermaid-sequence-actor-line-color)}.actor-man circle,.actor-man line{fill:var(--md-mermaid-sequence-actorman-bg-color);stroke:var(--md-mermaid-sequence-actorman-line-color)}.messageLine0,.messageLine1{stroke:var(--md-mermaid-sequence-message-line-color)}.note{fill:var(--md-mermaid-sequence-note-bg-color);stroke:var(--md-mermaid-sequence-note-border-color)}.loopText,.loopText>tspan,.messageText,.noteText>tspan{stroke:none;font-family:var(--md-mermaid-font-family)!important}.messageText{fill:var(--md-mermaid-sequence-message-fg-color)}.loopText,.loopText>tspan{fill:var(--md-mermaid-sequence-loop-fg-color)}.noteText>tspan{fill:var(--md-mermaid-sequence-note-fg-color)}#arrowhead path{fill:var(--md-mermaid-sequence-message-line-color);stroke:none}.loopLine{fill:var(--md-mermaid-sequence-loop-bg-color);stroke:var(--md-mermaid-sequence-loop-border-color)}.labelBox{fill:var(--md-mermaid-sequence-label-bg-color);stroke:none}.labelText,.labelText>span{fill:var(--md-mermaid-sequence-label-fg-color);font-family:var(--md-mermaid-font-family)}.sequenceNumber{fill:var(--md-mermaid-sequence-number-fg-color)}rect.rect{fill:var(--md-mermaid-sequence-box-bg-color);stroke:none}rect.rect+text.text{fill:var(--md-mermaid-sequence-box-fg-color)}defs #sequencenumber{fill:var(--md-mermaid-sequence-number-bg-color)!important}";var qr,ka=0;function Ha(){return typeof mermaid=="undefined"||mermaid instanceof Element?gt("https://unpkg.com/mermaid@10.7.0/dist/mermaid.min.js"):$(void 0)}function _n(e){return e.classList.remove("mermaid"),qr||(qr=Ha().pipe(y(()=>mermaid.initialize({startOnLoad:!1,themeCSS:Ln,sequence:{actorFontSize:"16px",messageFontSize:"16px",noteFontSize:"16px"}})),m(()=>{}),B(1))),qr.subscribe(()=>ro(this,null,function*(){e.classList.add("mermaid");let t=`__mermaid_${ka++}`,r=E("div",{class:"mermaid"}),o=e.textContent,{svg:n,fn:i}=yield mermaid.render(t,o),s=r.attachShadow({mode:"closed"});s.innerHTML=n,e.replaceWith(r),i==null||i(s)})),qr.pipe(m(()=>({ref:e})))}var An=E("table");function Cn(e){return e.replaceWith(An),An.replaceWith(vn(e)),$({ref:e})}function $a(e){let t=e.find(r=>r.checked)||e[0];return T(...e.map(r=>d(r,"change").pipe(m(()=>P(`label[for="${r.id}"]`))))).pipe(q(P(`label[for="${t.id}"]`)),m(r=>({active:r})))}function kn(e,{viewport$:t,target$:r}){let o=P(".tabbed-labels",e),n=R(":scope > input",e),i=Nr("prev");e.append(i);let s=Nr("next");return e.append(s),H(()=>{let a=new v,c=a.pipe(ee(),oe(!0));Q([a,Ee(e)]).pipe(U(c),Me(1,de)).subscribe({next([{active:p},l]){let f=Ue(p),{width:u}=pe(p);e.style.setProperty("--md-indicator-x",`${f.x}px`),e.style.setProperty("--md-indicator-width",`${u}px`);let h=ir(o);(f.x<h.x||f.x+u>h.x+l.width)&&o.scrollTo({left:Math.max(0,f.x-16),behavior:"smooth"})},complete(){e.style.removeProperty("--md-indicator-x"),e.style.removeProperty("--md-indicator-width")}}),Q([et(o),Ee(o)]).pipe(U(c)).subscribe(([p,l])=>{let f=xt(o);i.hidden=p.x<16,s.hidden=p.x>f.width-l.width-16}),T(d(i,"click").pipe(m(()=>-1)),d(s,"click").pipe(m(()=>1))).pipe(U(c)).subscribe(p=>{let{width:l}=pe(o);o.scrollBy({left:l*p,behavior:"smooth"})}),r.pipe(U(c),g(p=>n.includes(p))).subscribe(p=>p.click()),o.classList.add("tabbed-labels--linked");for(let p of n){let l=P(`label[for="${p.id}"]`);l.replaceChildren(E("a",{href:`#${l.htmlFor}`,tabIndex:-1},...Array.from(l.childNodes))),d(l.firstElementChild,"click").pipe(U(c),g(f=>!(f.metaKey||f.ctrlKey)),y(f=>{f.preventDefault(),f.stopPropagation()})).subscribe(()=>{history.replaceState({},"",`#${l.htmlFor}`),l.click()})}return G("content.tabs.link")&&a.pipe(Le(1),ae(t)).subscribe(([{active:p},{offset:l}])=>{let f=p.innerText.trim();if(p.hasAttribute("data-md-switching"))p.removeAttribute("data-md-switching");else{let u=e.offsetTop-l.y;for(let w of R("[data-tabs]"))for(let A of R(":scope > input",w)){let Z=P(`label[for="${A.id}"]`);if(Z!==p&&Z.innerText.trim()===f){Z.setAttribute("data-md-switching",""),A.click();break}}window.scrollTo({top:e.offsetTop-u});let h=__md_get("__tabs")||[];__md_set("__tabs",[...new Set([f,...h])])}}),a.pipe(U(c)).subscribe(()=>{for(let p of R("audio, video",e))p.pause()}),$a(n).pipe(y(p=>a.next(p)),_(()=>a.complete()),m(p=>F({ref:e},p)))}).pipe(ze(ie))}function Hn(e,{viewport$:t,target$:r,print$:o}){return T(...R(".annotate:not(.highlight)",e).map(n=>wn(n,{target$:r,print$:o})),...R("pre:not(.mermaid) > code",e).map(n=>On(n,{target$:r,print$:o})),...R("pre.mermaid",e).map(n=>_n(n)),...R("table:not([class])",e).map(n=>Cn(n)),...R("details",e).map(n=>Mn(n,{target$:r,print$:o})),...R("[data-tabs]",e).map(n=>kn(n,{viewport$:t,target$:r})),...R("[title]",e).filter(()=>G("content.tooltips")).map(n=>Ge(n)))}function Ra(e,{alert$:t}){return t.pipe(b(r=>T($(!0),$(!1).pipe(Ye(2e3))).pipe(m(o=>({message:r,active:o})))))}function $n(e,t){let r=P(".md-typeset",e);return H(()=>{let o=new v;return o.subscribe(({message:n,active:i})=>{e.classList.toggle("md-dialog--active",i),r.textContent=n}),Ra(e,t).pipe(y(n=>o.next(n)),_(()=>o.complete()),m(n=>F({ref:e},n)))})}function Pa({viewport$:e}){if(!G("header.autohide"))return $(!1);let t=e.pipe(m(({offset:{y:n}})=>n),Ke(2,1),m(([n,i])=>[n<i,i]),X(0)),r=Q([e,t]).pipe(g(([{offset:n},[,i]])=>Math.abs(i-n.y)>100),m(([,[n]])=>n),Y()),o=We("search");return Q([e,o]).pipe(m(([{offset:n},i])=>n.y>400&&!i),Y(),b(n=>n?r:$(!1)),q(!1))}function Rn(e,t){return H(()=>Q([Ee(e),Pa(t)])).pipe(m(([{height:r},o])=>({height:r,hidden:o})),Y((r,o)=>r.height===o.height&&r.hidden===o.hidden),B(1))}function Pn(e,{header$:t,main$:r}){return H(()=>{let o=new v,n=o.pipe(ee(),oe(!0));o.pipe(X("active"),je(t)).subscribe(([{active:s},{hidden:a}])=>{e.classList.toggle("md-header--shadow",s&&!a),e.hidden=a});let i=fe(R("[title]",e)).pipe(g(()=>G("content.tooltips")),re(s=>Ge(s)));return r.subscribe(o),t.pipe(U(n),m(s=>F({ref:e},s)),$e(i.pipe(U(n))))})}function Ia(e,{viewport$:t,header$:r}){return pr(e,{viewport$:t,header$:r}).pipe(m(({offset:{y:o}})=>{let{height:n}=pe(e);return{active:o>=n}}),X("active"))}function In(e,t){return H(()=>{let r=new v;r.subscribe({next({active:n}){e.classList.toggle("md-header__title--active",n)},complete(){e.classList.remove("md-header__title--active")}});let o=me(".md-content h1");return typeof o=="undefined"?L:Ia(o,t).pipe(y(n=>r.next(n)),_(()=>r.complete()),m(n=>F({ref:e},n)))})}function Fn(e,{viewport$:t,header$:r}){let o=r.pipe(m(({height:i})=>i),Y()),n=o.pipe(b(()=>Ee(e).pipe(m(({height:i})=>({top:e.offsetTop,bottom:e.offsetTop+i})),X("bottom"))));return Q([o,n,t]).pipe(m(([i,{top:s,bottom:a},{offset:{y:c},size:{height:p}}])=>(p=Math.max(0,p-Math.max(0,s-c,i)-Math.max(0,p+c-a)),{offset:s-i,height:p,active:s-i<=c})),Y((i,s)=>i.offset===s.offset&&i.height===s.height&&i.active===s.active))}function Fa(e){let t=__md_get("__palette")||{index:e.findIndex(o=>matchMedia(o.getAttribute("data-md-color-media")).matches)},r=Math.max(0,Math.min(t.index,e.length-1));return $(...e).pipe(re(o=>d(o,"change").pipe(m(()=>o))),q(e[r]),m(o=>({index:e.indexOf(o),color:{media:o.getAttribute("data-md-color-media"),scheme:o.getAttribute("data-md-color-scheme"),primary:o.getAttribute("data-md-color-primary"),accent:o.getAttribute("data-md-color-accent")}})),B(1))}function jn(e){let t=R("input",e),r=E("meta",{name:"theme-color"});document.head.appendChild(r);let o=E("meta",{name:"color-scheme"});document.head.appendChild(o);let n=At("(prefers-color-scheme: light)");return H(()=>{let i=new v;return i.subscribe(s=>{if(document.body.setAttribute("data-md-color-switching",""),s.color.media==="(prefers-color-scheme)"){let a=matchMedia("(prefers-color-scheme: light)"),c=document.querySelector(a.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");s.color.scheme=c.getAttribute("data-md-color-scheme"),s.color.primary=c.getAttribute("data-md-color-primary"),s.color.accent=c.getAttribute("data-md-color-accent")}for(let[a,c]of Object.entries(s.color))document.body.setAttribute(`data-md-color-${a}`,c);for(let a=0;a<t.length;a++){let c=t[a].nextElementSibling;c instanceof HTMLElement&&(c.hidden=s.index!==a)}__md_set("__palette",s)}),i.pipe(m(()=>{let s=Te("header"),a=window.getComputedStyle(s);return o.content=a.colorScheme,a.backgroundColor.match(/\d+/g).map(c=>(+c).toString(16).padStart(2,"0")).join("")})).subscribe(s=>r.content=`#${s}`),i.pipe(Oe(ie)).subscribe(()=>{document.body.removeAttribute("data-md-color-switching")}),Fa(t).pipe(U(n.pipe(Le(1))),at(),y(s=>i.next(s)),_(()=>i.complete()),m(s=>F({ref:e},s)))})}function Un(e,{progress$:t}){return H(()=>{let r=new v;return r.subscribe(({value:o})=>{e.style.setProperty("--md-progress-value",`${o}`)}),t.pipe(y(o=>r.next({value:o})),_(()=>r.complete()),m(o=>({ref:e,value:o})))})}var Kr=jt(zr());function ja(e){e.setAttribute("data-md-copying","");let t=e.closest("[data-copy]"),r=t?t.getAttribute("data-copy"):e.innerText;return e.removeAttribute("data-md-copying"),r.trimEnd()}function Wn({alert$:e}){Kr.default.isSupported()&&new j(t=>{new Kr.default("[data-clipboard-target], [data-clipboard-text]",{text:r=>r.getAttribute("data-clipboard-text")||ja(P(r.getAttribute("data-clipboard-target")))}).on("success",r=>t.next(r))}).pipe(y(t=>{t.trigger.focus()}),m(()=>ge("clipboard.copied"))).subscribe(e)}function Dn(e,t){return e.protocol=t.protocol,e.hostname=t.hostname,e}function Ua(e,t){let r=new Map;for(let o of R("url",e)){let n=P("loc",o),i=[Dn(new URL(n.textContent),t)];r.set(`${i[0]}`,i);for(let s of R("[rel=alternate]",o)){let a=s.getAttribute("href");a!=null&&i.push(Dn(new URL(a),t))}}return r}function mr(e){return on(new URL("sitemap.xml",e)).pipe(m(t=>Ua(t,new URL(e))),he(()=>$(new Map)))}function Wa(e,t){if(!(e.target instanceof Element))return L;let r=e.target.closest("a");if(r===null)return L;if(r.target||e.metaKey||e.ctrlKey)return L;let o=new URL(r.href);return o.search=o.hash="",t.has(`${o}`)?(e.preventDefault(),$(new URL(r.href))):L}function Nn(e){let t=new Map;for(let r of R(":scope > *",e.head))t.set(r.outerHTML,r);return t}function Vn(e){for(let t of R("[href], [src]",e))for(let r of["href","src"]){let o=t.getAttribute(r);if(o&&!/^(?:[a-z]+:)?\/\//i.test(o)){t[r]=t[r];break}}return $(e)}function Da(e){for(let o of["[data-md-component=announce]","[data-md-component=container]","[data-md-component=header-topic]","[data-md-component=outdated]","[data-md-component=logo]","[data-md-component=skip]",...G("navigation.tabs.sticky")?["[data-md-component=tabs]"]:[]]){let n=me(o),i=me(o,e);typeof n!="undefined"&&typeof i!="undefined"&&n.replaceWith(i)}let t=Nn(document);for(let[o,n]of Nn(e))t.has(o)?t.delete(o):document.head.appendChild(n);for(let o of t.values()){let n=o.getAttribute("name");n!=="theme-color"&&n!=="color-scheme"&&o.remove()}let r=Te("container");return Fe(R("script",r)).pipe(b(o=>{let n=e.createElement("script");if(o.src){for(let i of o.getAttributeNames())n.setAttribute(i,o.getAttribute(i));return o.replaceWith(n),new j(i=>{n.onload=()=>i.complete()})}else return n.textContent=o.textContent,o.replaceWith(n),L}),ee(),oe(document))}function zn({location$:e,viewport$:t,progress$:r}){let o=we();if(location.protocol==="file:")return L;let n=mr(o.base);$(document).subscribe(Vn);let i=d(document.body,"click").pipe(je(n),b(([c,p])=>Wa(c,p)),le()),s=d(window,"popstate").pipe(m(ve),le());i.pipe(ae(t)).subscribe(([c,{offset:p}])=>{history.replaceState(p,""),history.pushState(null,"",c)}),T(i,s).subscribe(e);let a=e.pipe(X("pathname"),b(c=>rn(c,{progress$:r}).pipe(he(()=>(st(c,!0),L)))),b(Vn),b(Da),le());return T(a.pipe(ae(e,(c,p)=>p)),e.pipe(X("pathname"),b(()=>e),X("hash")),e.pipe(Y((c,p)=>c.pathname===p.pathname&&c.hash===p.hash),b(()=>i),y(()=>history.back()))).subscribe(c=>{var p,l;history.state!==null||!c.hash?window.scrollTo(0,(l=(p=history.state)==null?void 0:p.y)!=null?l:0):(history.scrollRestoration="auto",Zo(c.hash),history.scrollRestoration="manual")}),e.subscribe(()=>{history.scrollRestoration="manual"}),d(window,"beforeunload").subscribe(()=>{history.scrollRestoration="auto"}),t.pipe(X("offset"),be(100)).subscribe(({offset:c})=>{history.replaceState(c,"")}),a}var Qn=jt(Kn());function Yn(e){let t=e.separator.split("|").map(n=>n.replace(/(\(\?[!=<][^)]+\))/g,"").length===0?"\uFFFD":n).join("|"),r=new RegExp(t,"img"),o=(n,i,s)=>`${i}<mark data-md-highlight>${s}</mark>`;return n=>{n=n.replace(/[\s*+\-:~^]+/g," ").trim();let i=new RegExp(`(^|${e.separator}|)(${n.replace(/[|\\{}()[\]^$+*?.-]/g,"\\$&").replace(r,"|")})`,"img");return s=>(0,Qn.default)(s).replace(i,o).replace(/<\/mark>(\s+)<mark[^>]*>/img,"$1")}}function Ht(e){return e.type===1}function fr(e){return e.type===3}function Bn(e,t){let r=ln(e);return T($(location.protocol!=="file:"),We("search")).pipe(He(o=>o),b(()=>t)).subscribe(({config:o,docs:n})=>r.next({type:0,data:{config:o,docs:n,options:{suggest:G("search.suggest")}}})),r}function Gn({document$:e}){let t=we(),r=De(new URL("../versions.json",t.base)).pipe(he(()=>L)),o=r.pipe(m(n=>{let[,i]=t.base.match(/([^/]+)\/?$/);return n.find(({version:s,aliases:a})=>s===i||a.includes(i))||n[0]}));r.pipe(m(n=>new Map(n.map(i=>[`${new URL(`../${i.version}/`,t.base)}`,i]))),b(n=>d(document.body,"click").pipe(g(i=>!i.metaKey&&!i.ctrlKey),ae(o),b(([i,s])=>{if(i.target instanceof Element){let a=i.target.closest("a");if(a&&!a.target&&n.has(a.href)){let c=a.href;return!i.target.closest(".md-version")&&n.get(c)===s?L:(i.preventDefault(),$(c))}}return L}),b(i=>{let{version:s}=n.get(i);return mr(new URL(i)).pipe(m(a=>{let p=ve().href.replace(t.base,"");return a.has(p.split("#")[0])?new URL(`../${s}/${p}`,t.base):new URL(i)}))})))).subscribe(n=>st(n,!0)),Q([r,o]).subscribe(([n,i])=>{P(".md-header__topic").appendChild(gn(n,i))}),e.pipe(b(()=>o)).subscribe(n=>{var s;let i=__md_get("__outdated",sessionStorage);if(i===null){i=!0;let a=((s=t.version)==null?void 0:s.default)||"latest";Array.isArray(a)||(a=[a]);e:for(let c of a)for(let p of n.aliases.concat(n.version))if(new RegExp(c,"i").test(p)){i=!1;break e}__md_set("__outdated",i,sessionStorage)}if(i)for(let a of ne("outdated"))a.hidden=!1})}function Ka(e,{worker$:t}){let{searchParams:r}=ve();r.has("q")&&(Be("search",!0),e.value=r.get("q"),e.focus(),We("search").pipe(He(i=>!i)).subscribe(()=>{let i=ve();i.searchParams.delete("q"),history.replaceState({},"",`${i}`)}));let o=vt(e),n=T(t.pipe(He(Ht)),d(e,"keyup"),o).pipe(m(()=>e.value),Y());return Q([n,o]).pipe(m(([i,s])=>({value:i,focus:s})),B(1))}function Jn(e,{worker$:t}){let r=new v,o=r.pipe(ee(),oe(!0));Q([t.pipe(He(Ht)),r],(i,s)=>s).pipe(X("value")).subscribe(({value:i})=>t.next({type:2,data:i})),r.pipe(X("focus")).subscribe(({focus:i})=>{i&&Be("search",i)}),d(e.form,"reset").pipe(U(o)).subscribe(()=>e.focus());let n=P("header [for=__search]");return d(n,"click").subscribe(()=>e.focus()),Ka(e,{worker$:t}).pipe(y(i=>r.next(i)),_(()=>r.complete()),m(i=>F({ref:e},i)),B(1))}function Xn(e,{worker$:t,query$:r}){let o=new v,n=Yo(e.parentElement).pipe(g(Boolean)),i=e.parentElement,s=P(":scope > :first-child",e),a=P(":scope > :last-child",e);We("search").subscribe(l=>a.setAttribute("role",l?"list":"presentation")),o.pipe(ae(r),Ir(t.pipe(He(Ht)))).subscribe(([{items:l},{value:f}])=>{switch(l.length){case 0:s.textContent=f.length?ge("search.result.none"):ge("search.result.placeholder");break;case 1:s.textContent=ge("search.result.one");break;default:let u=ar(l.length);s.textContent=ge("search.result.other",u)}});let c=o.pipe(y(()=>a.innerHTML=""),b(({items:l})=>T($(...l.slice(0,10)),$(...l.slice(10)).pipe(Ke(4),jr(n),b(([f])=>f)))),m(hn),le());return c.subscribe(l=>a.appendChild(l)),c.pipe(re(l=>{let f=me("details",l);return typeof f=="undefined"?L:d(f,"toggle").pipe(U(o),m(()=>f))})).subscribe(l=>{l.open===!1&&l.offsetTop<=i.scrollTop&&i.scrollTo({top:l.offsetTop})}),t.pipe(g(fr),m(({data:l})=>l)).pipe(y(l=>o.next(l)),_(()=>o.complete()),m(l=>F({ref:e},l)))}function Qa(e,{query$:t}){return t.pipe(m(({value:r})=>{let o=ve();return o.hash="",r=r.replace(/\s+/g,"+").replace(/&/g,"%26").replace(/=/g,"%3D"),o.search=`q=${r}`,{url:o}}))}function Zn(e,t){let r=new v,o=r.pipe(ee(),oe(!0));return r.subscribe(({url:n})=>{e.setAttribute("data-clipboard-text",e.href),e.href=`${n}`}),d(e,"click").pipe(U(o)).subscribe(n=>n.preventDefault()),Qa(e,t).pipe(y(n=>r.next(n)),_(()=>r.complete()),m(n=>F({ref:e},n)))}function ei(e,{worker$:t,keyboard$:r}){let o=new v,n=Te("search-query"),i=T(d(n,"keydown"),d(n,"focus")).pipe(Oe(ie),m(()=>n.value),Y());return o.pipe(je(i),m(([{suggest:a},c])=>{let p=c.split(/([\s-]+)/);if(a!=null&&a.length&&p[p.length-1]){let l=a[a.length-1];l.startsWith(p[p.length-1])&&(p[p.length-1]=l)}else p.length=0;return p})).subscribe(a=>e.innerHTML=a.join("").replace(/\s/g,"&nbsp;")),r.pipe(g(({mode:a})=>a==="search")).subscribe(a=>{switch(a.type){case"ArrowRight":e.innerText.length&&n.selectionStart===n.value.length&&(n.value=e.innerText);break}}),t.pipe(g(fr),m(({data:a})=>a)).pipe(y(a=>o.next(a)),_(()=>o.complete()),m(()=>({ref:e})))}function ti(e,{index$:t,keyboard$:r}){let o=we();try{let n=Bn(o.search,t),i=Te("search-query",e),s=Te("search-result",e);d(e,"click").pipe(g(({target:c})=>c instanceof Element&&!!c.closest("a"))).subscribe(()=>Be("search",!1)),r.pipe(g(({mode:c})=>c==="search")).subscribe(c=>{let p=Re();switch(c.type){case"Enter":if(p===i){let l=new Map;for(let f of R(":first-child [href]",s)){let u=f.firstElementChild;l.set(f,parseFloat(u.getAttribute("data-md-score")))}if(l.size){let[[f]]=[...l].sort(([,u],[,h])=>h-u);f.click()}c.claim()}break;case"Escape":case"Tab":Be("search",!1),i.blur();break;case"ArrowUp":case"ArrowDown":if(typeof p=="undefined")i.focus();else{let l=[i,...R(":not(details) > [href], summary, details[open] [href]",s)],f=Math.max(0,(Math.max(0,l.indexOf(p))+l.length+(c.type==="ArrowUp"?-1:1))%l.length);l[f].focus()}c.claim();break;default:i!==Re()&&i.focus()}}),r.pipe(g(({mode:c})=>c==="global")).subscribe(c=>{switch(c.type){case"f":case"s":case"/":i.focus(),i.select(),c.claim();break}});let a=Jn(i,{worker$:n});return T(a,Xn(s,{worker$:n,query$:a})).pipe($e(...ne("search-share",e).map(c=>Zn(c,{query$:a})),...ne("search-suggest",e).map(c=>ei(c,{worker$:n,keyboard$:r}))))}catch(n){return e.hidden=!0,qe}}function ri(e,{index$:t,location$:r}){return Q([t,r.pipe(q(ve()),g(o=>!!o.searchParams.get("h")))]).pipe(m(([o,n])=>Yn(o.config)(n.searchParams.get("h"))),m(o=>{var s;let n=new Map,i=document.createNodeIterator(e,NodeFilter.SHOW_TEXT);for(let a=i.nextNode();a;a=i.nextNode())if((s=a.parentElement)!=null&&s.offsetHeight){let c=a.textContent,p=o(c);p.length>c.length&&n.set(a,p)}for(let[a,c]of n){let{childNodes:p}=E("span",null,c);a.replaceWith(...Array.from(p))}return{ref:e,nodes:n}}))}function Ya(e,{viewport$:t,main$:r}){let o=e.closest(".md-grid"),n=o.offsetTop-o.parentElement.offsetTop;return Q([r,t]).pipe(m(([{offset:i,height:s},{offset:{y:a}}])=>(s=s+Math.min(n,Math.max(0,a-i))-n,{height:s,locked:a>=i+n})),Y((i,s)=>i.height===s.height&&i.locked===s.locked))}function Qr(e,o){var n=o,{header$:t}=n,r=to(n,["header$"]);let i=P(".md-sidebar__scrollwrap",e),{y:s}=Ue(i);return H(()=>{let a=new v,c=a.pipe(ee(),oe(!0)),p=a.pipe(Me(0,de));return p.pipe(ae(t)).subscribe({next([{height:l},{height:f}]){i.style.height=`${l-2*s}px`,e.style.top=`${f}px`},complete(){i.style.height="",e.style.top=""}}),p.pipe(He()).subscribe(()=>{for(let l of R(".md-nav__link--active[href]",e)){if(!l.clientHeight)continue;let f=l.closest(".md-sidebar__scrollwrap");if(typeof f!="undefined"){let u=l.offsetTop-f.offsetTop,{height:h}=pe(f);f.scrollTo({top:u-h/2})}}}),fe(R("label[tabindex]",e)).pipe(re(l=>d(l,"click").pipe(Oe(ie),m(()=>l),U(c)))).subscribe(l=>{let f=P(`[id="${l.htmlFor}"]`);P(`[aria-labelledby="${l.id}"]`).setAttribute("aria-expanded",`${f.checked}`)}),Ya(e,r).pipe(y(l=>a.next(l)),_(()=>a.complete()),m(l=>F({ref:e},l)))})}function oi(e,t){if(typeof t!="undefined"){let r=`https://api.github.com/repos/${e}/${t}`;return Lt(De(`${r}/releases/latest`).pipe(he(()=>L),m(o=>({version:o.tag_name})),Qe({})),De(r).pipe(he(()=>L),m(o=>({stars:o.stargazers_count,forks:o.forks_count})),Qe({}))).pipe(m(([o,n])=>F(F({},o),n)))}else{let r=`https://api.github.com/users/${e}`;return De(r).pipe(m(o=>({repositories:o.public_repos})),Qe({}))}}function ni(e,t){let r=`https://${e}/api/v4/projects/${encodeURIComponent(t)}`;return De(r).pipe(he(()=>L),m(({star_count:o,forks_count:n})=>({stars:o,forks:n})),Qe({}))}function ii(e){let t=e.match(/^.+github\.com\/([^/]+)\/?([^/]+)?/i);if(t){let[,r,o]=t;return oi(r,o)}if(t=e.match(/^.+?([^/]*gitlab[^/]+)\/(.+?)\/?$/i),t){let[,r,o]=t;return ni(r,o)}return L}var Ba;function Ga(e){return Ba||(Ba=H(()=>{let t=__md_get("__source",sessionStorage);if(t)return $(t);if(ne("consent").length){let o=__md_get("__consent");if(!(o&&o.github))return L}return ii(e.href).pipe(y(o=>__md_set("__source",o,sessionStorage)))}).pipe(he(()=>L),g(t=>Object.keys(t).length>0),m(t=>({facts:t})),B(1)))}function ai(e){let t=P(":scope > :last-child",e);return H(()=>{let r=new v;return r.subscribe(({facts:o})=>{t.appendChild(bn(o)),t.classList.add("md-source__repository--active")}),Ga(e).pipe(y(o=>r.next(o)),_(()=>r.complete()),m(o=>F({ref:e},o)))})}function Ja(e,{viewport$:t,header$:r}){return Ee(document.body).pipe(b(()=>pr(e,{header$:r,viewport$:t})),m(({offset:{y:o}})=>({hidden:o>=10})),X("hidden"))}function si(e,t){return H(()=>{let r=new v;return r.subscribe({next({hidden:o}){e.hidden=o},complete(){e.hidden=!1}}),(G("navigation.tabs.sticky")?$({hidden:!1}):Ja(e,t)).pipe(y(o=>r.next(o)),_(()=>r.complete()),m(o=>F({ref:e},o)))})}function Xa(e,{viewport$:t,header$:r}){let o=new Map,n=R(".md-nav__link",e);for(let a of n){let c=decodeURIComponent(a.hash.substring(1)),p=me(`[id="${c}"]`);typeof p!="undefined"&&o.set(a,p)}let i=r.pipe(X("height"),m(({height:a})=>{let c=Te("main"),p=P(":scope > :first-child",c);return a+.8*(p.offsetTop-c.offsetTop)}),le());return Ee(document.body).pipe(X("height"),b(a=>H(()=>{let c=[];return $([...o].reduce((p,[l,f])=>{for(;c.length&&o.get(c[c.length-1]).tagName>=f.tagName;)c.pop();let u=f.offsetTop;for(;!u&&f.parentElement;)f=f.parentElement,u=f.offsetTop;let h=f.offsetParent;for(;h;h=h.offsetParent)u+=h.offsetTop;return p.set([...c=[...c,l]].reverse(),u)},new Map))}).pipe(m(c=>new Map([...c].sort(([,p],[,l])=>p-l))),je(i),b(([c,p])=>t.pipe(Rr(([l,f],{offset:{y:u},size:h})=>{let w=u+h.height>=Math.floor(a.height);for(;f.length;){let[,A]=f[0];if(A-p<u||w)l=[...l,f.shift()];else break}for(;l.length;){let[,A]=l[l.length-1];if(A-p>=u&&!w)f=[l.pop(),...f];else break}return[l,f]},[[],[...c]]),Y((l,f)=>l[0]===f[0]&&l[1]===f[1])))))).pipe(m(([a,c])=>({prev:a.map(([p])=>p),next:c.map(([p])=>p)})),q({prev:[],next:[]}),Ke(2,1),m(([a,c])=>a.prev.length<c.prev.length?{prev:c.prev.slice(Math.max(0,a.prev.length-1),c.prev.length),next:[]}:{prev:c.prev.slice(-1),next:c.next.slice(0,c.next.length-a.next.length)}))}function ci(e,{viewport$:t,header$:r,main$:o,target$:n}){return H(()=>{let i=new v,s=i.pipe(ee(),oe(!0));if(i.subscribe(({prev:a,next:c})=>{for(let[p]of c)p.classList.remove("md-nav__link--passed"),p.classList.remove("md-nav__link--active");for(let[p,[l]]of a.entries())l.classList.add("md-nav__link--passed"),l.classList.toggle("md-nav__link--active",p===a.length-1)}),G("toc.follow")){let a=T(t.pipe(be(1),m(()=>{})),t.pipe(be(250),m(()=>"smooth")));i.pipe(g(({prev:c})=>c.length>0),je(o.pipe(Oe(ie))),ae(a)).subscribe(([[{prev:c}],p])=>{let[l]=c[c.length-1];if(l.offsetHeight){let f=sr(l);if(typeof f!="undefined"){let u=l.offsetTop-f.offsetTop,{height:h}=pe(f);f.scrollTo({top:u-h/2,behavior:p})}}})}return G("navigation.tracking")&&t.pipe(U(s),X("offset"),be(250),Le(1),U(n.pipe(Le(1))),at({delay:250}),ae(i)).subscribe(([,{prev:a}])=>{let c=ve(),p=a[a.length-1];if(p&&p.length){let[l]=p,{hash:f}=new URL(l.href);c.hash!==f&&(c.hash=f,history.replaceState({},"",`${c}`))}else c.hash="",history.replaceState({},"",`${c}`)}),Xa(e,{viewport$:t,header$:r}).pipe(y(a=>i.next(a)),_(()=>i.complete()),m(a=>F({ref:e},a)))})}function Za(e,{viewport$:t,main$:r,target$:o}){let n=t.pipe(m(({offset:{y:s}})=>s),Ke(2,1),m(([s,a])=>s>a&&a>0),Y()),i=r.pipe(m(({active:s})=>s));return Q([i,n]).pipe(m(([s,a])=>!(s&&a)),Y(),U(o.pipe(Le(1))),oe(!0),at({delay:250}),m(s=>({hidden:s})))}function pi(e,{viewport$:t,header$:r,main$:o,target$:n}){let i=new v,s=i.pipe(ee(),oe(!0));return i.subscribe({next({hidden:a}){e.hidden=a,a?(e.setAttribute("tabindex","-1"),e.blur()):e.removeAttribute("tabindex")},complete(){e.style.top="",e.hidden=!0,e.removeAttribute("tabindex")}}),r.pipe(U(s),X("height")).subscribe(({height:a})=>{e.style.top=`${a+16}px`}),d(e,"click").subscribe(a=>{a.preventDefault(),window.scrollTo({top:0})}),Za(e,{viewport$:t,main$:o,target$:n}).pipe(y(a=>i.next(a)),_(()=>i.complete()),m(a=>F({ref:e},a)))}function li({document$:e}){e.pipe(b(()=>R(".md-ellipsis")),re(t=>yt(t).pipe(U(e.pipe(Le(1))),g(r=>r),m(()=>t),ye(1))),g(t=>t.offsetWidth<t.scrollWidth),re(t=>{let r=t.innerText,o=t.closest("a")||t;return o.title=r,Ge(o).pipe(U(e.pipe(Le(1))),_(()=>o.removeAttribute("title")))})).subscribe(),e.pipe(b(()=>R(".md-status")),re(t=>Ge(t))).subscribe()}function mi({document$:e,tablet$:t}){e.pipe(b(()=>R(".md-toggle--indeterminate")),y(r=>{r.indeterminate=!0,r.checked=!1}),re(r=>d(r,"change").pipe(Fr(()=>r.classList.contains("md-toggle--indeterminate")),m(()=>r))),ae(t)).subscribe(([r,o])=>{r.classList.remove("md-toggle--indeterminate"),o&&(r.checked=!1)})}function es(){return/(iPad|iPhone|iPod)/.test(navigator.userAgent)}function fi({document$:e}){e.pipe(b(()=>R("[data-md-scrollfix]")),y(t=>t.removeAttribute("data-md-scrollfix")),g(es),re(t=>d(t,"touchstart").pipe(m(()=>t)))).subscribe(t=>{let r=t.scrollTop;r===0?t.scrollTop=1:r+t.offsetHeight===t.scrollHeight&&(t.scrollTop=r-1)})}function ui({viewport$:e,tablet$:t}){Q([We("search"),t]).pipe(m(([r,o])=>r&&!o),b(r=>$(r).pipe(Ye(r?400:100))),ae(e)).subscribe(([r,{offset:{y:o}}])=>{if(r)document.body.setAttribute("data-md-scrolllock",""),document.body.style.top=`-${o}px`;else{let n=-1*parseInt(document.body.style.top,10);document.body.removeAttribute("data-md-scrolllock"),document.body.style.top="",n&&window.scrollTo(0,n)}})}Object.entries||(Object.entries=function(e){let t=[];for(let r of Object.keys(e))t.push([r,e[r]]);return t});Object.values||(Object.values=function(e){let t=[];for(let r of Object.keys(e))t.push(e[r]);return t});typeof Element!="undefined"&&(Element.prototype.scrollTo||(Element.prototype.scrollTo=function(e,t){typeof e=="object"?(this.scrollLeft=e.left,this.scrollTop=e.top):(this.scrollLeft=e,this.scrollTop=t)}),Element.prototype.replaceWith||(Element.prototype.replaceWith=function(...e){let t=this.parentNode;if(t){e.length===0&&t.removeChild(this);for(let r=e.length-1;r>=0;r--){let o=e[r];typeof o=="string"?o=document.createTextNode(o):o.parentNode&&o.parentNode.removeChild(o),r?t.insertBefore(this.previousSibling,o):t.replaceChild(o,this)}}}));function ts(){return location.protocol==="file:"?gt(`${new URL("search/search_index.js",Yr.base)}`).pipe(m(()=>__index),B(1)):De(new URL("search/search_index.json",Yr.base))}document.documentElement.classList.remove("no-js");document.documentElement.classList.add("js");var rt=No(),Rt=Jo(),wt=en(Rt),Br=Go(),_e=pn(),ur=At("(min-width: 960px)"),hi=At("(min-width: 1220px)"),bi=tn(),Yr=we(),vi=document.forms.namedItem("search")?ts():qe,Gr=new v;Wn({alert$:Gr});var Jr=new v;G("navigation.instant")&&zn({location$:Rt,viewport$:_e,progress$:Jr}).subscribe(rt);var di;((di=Yr.version)==null?void 0:di.provider)==="mike"&&Gn({document$:rt});T(Rt,wt).pipe(Ye(125)).subscribe(()=>{Be("drawer",!1),Be("search",!1)});Br.pipe(g(({mode:e})=>e==="global")).subscribe(e=>{switch(e.type){case"p":case",":let t=me("link[rel=prev]");typeof t!="undefined"&&st(t);break;case"n":case".":let r=me("link[rel=next]");typeof r!="undefined"&&st(r);break;case"Enter":let o=Re();o instanceof HTMLLabelElement&&o.click()}});li({document$:rt});mi({document$:rt,tablet$:ur});fi({document$:rt});ui({viewport$:_e,tablet$:ur});var tt=Rn(Te("header"),{viewport$:_e}),$t=rt.pipe(m(()=>Te("main")),b(e=>Fn(e,{viewport$:_e,header$:tt})),B(1)),rs=T(...ne("consent").map(e=>fn(e,{target$:wt})),...ne("dialog").map(e=>$n(e,{alert$:Gr})),...ne("header").map(e=>Pn(e,{viewport$:_e,header$:tt,main$:$t})),...ne("palette").map(e=>jn(e)),...ne("progress").map(e=>Un(e,{progress$:Jr})),...ne("search").map(e=>ti(e,{index$:vi,keyboard$:Br})),...ne("source").map(e=>ai(e))),os=H(()=>T(...ne("announce").map(e=>mn(e)),...ne("content").map(e=>Hn(e,{viewport$:_e,target$:wt,print$:bi})),...ne("content").map(e=>G("search.highlight")?ri(e,{index$:vi,location$:Rt}):L),...ne("header-title").map(e=>In(e,{viewport$:_e,header$:tt})),...ne("sidebar").map(e=>e.getAttribute("data-md-type")==="navigation"?Ur(hi,()=>Qr(e,{viewport$:_e,header$:tt,main$:$t})):Ur(ur,()=>Qr(e,{viewport$:_e,header$:tt,main$:$t}))),...ne("tabs").map(e=>si(e,{viewport$:_e,header$:tt})),...ne("toc").map(e=>ci(e,{viewport$:_e,header$:tt,main$:$t,target$:wt})),...ne("top").map(e=>pi(e,{viewport$:_e,header$:tt,main$:$t,target$:wt})))),gi=rt.pipe(b(()=>os),$e(rs),B(1));gi.subscribe();window.document$=rt;window.location$=Rt;window.target$=wt;window.keyboard$=Br;window.viewport$=_e;window.tablet$=ur;window.screen$=hi;window.print$=bi;window.alert$=Gr;window.progress$=Jr;window.component$=gi;})();
+//# sourceMappingURL=bundle.c8d2eff1.min.js.map
+
diff --git a/assets/javascripts/bundle.c8d2eff1.min.js.map b/assets/javascripts/bundle.c8d2eff1.min.js.map
new file mode 100644
index 000000000..fc522dbae
--- /dev/null
+++ b/assets/javascripts/bundle.c8d2eff1.min.js.map
@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["node_modules/focus-visible/dist/focus-visible.js", "node_modules/clipboard/dist/clipboard.js", "node_modules/escape-html/index.js", "src/templates/assets/javascripts/bundle.ts", "node_modules/rxjs/node_modules/tslib/tslib.es6.js", "node_modules/rxjs/src/internal/util/isFunction.ts", "node_modules/rxjs/src/internal/util/createErrorClass.ts", "node_modules/rxjs/src/internal/util/UnsubscriptionError.ts", "node_modules/rxjs/src/internal/util/arrRemove.ts", "node_modules/rxjs/src/internal/Subscription.ts", "node_modules/rxjs/src/internal/config.ts", "node_modules/rxjs/src/internal/scheduler/timeoutProvider.ts", "node_modules/rxjs/src/internal/util/reportUnhandledError.ts", "node_modules/rxjs/src/internal/util/noop.ts", "node_modules/rxjs/src/internal/NotificationFactories.ts", "node_modules/rxjs/src/internal/util/errorContext.ts", "node_modules/rxjs/src/internal/Subscriber.ts", "node_modules/rxjs/src/internal/symbol/observable.ts", "node_modules/rxjs/src/internal/util/identity.ts", "node_modules/rxjs/src/internal/util/pipe.ts", "node_modules/rxjs/src/internal/Observable.ts", "node_modules/rxjs/src/internal/util/lift.ts", "node_modules/rxjs/src/internal/operators/OperatorSubscriber.ts", "node_modules/rxjs/src/internal/scheduler/animationFrameProvider.ts", "node_modules/rxjs/src/internal/util/ObjectUnsubscribedError.ts", "node_modules/rxjs/src/internal/Subject.ts", "node_modules/rxjs/src/internal/scheduler/dateTimestampProvider.ts", "node_modules/rxjs/src/internal/ReplaySubject.ts", "node_modules/rxjs/src/internal/scheduler/Action.ts", "node_modules/rxjs/src/internal/scheduler/intervalProvider.ts", "node_modules/rxjs/src/internal/scheduler/AsyncAction.ts", "node_modules/rxjs/src/internal/Scheduler.ts", "node_modules/rxjs/src/internal/scheduler/AsyncScheduler.ts", "node_modules/rxjs/src/internal/scheduler/async.ts", "node_modules/rxjs/src/internal/scheduler/AnimationFrameAction.ts", "node_modules/rxjs/src/internal/scheduler/AnimationFrameScheduler.ts", "node_modules/rxjs/src/internal/scheduler/animationFrame.ts", "node_modules/rxjs/src/internal/observable/empty.ts", "node_modules/rxjs/src/internal/util/isScheduler.ts", "node_modules/rxjs/src/internal/util/args.ts", "node_modules/rxjs/src/internal/util/isArrayLike.ts", "node_modules/rxjs/src/internal/util/isPromise.ts", "node_modules/rxjs/src/internal/util/isInteropObservable.ts", "node_modules/rxjs/src/internal/util/isAsyncIterable.ts", "node_modules/rxjs/src/internal/util/throwUnobservableError.ts", "node_modules/rxjs/src/internal/symbol/iterator.ts", "node_modules/rxjs/src/internal/util/isIterable.ts", "node_modules/rxjs/src/internal/util/isReadableStreamLike.ts", "node_modules/rxjs/src/internal/observable/innerFrom.ts", "node_modules/rxjs/src/internal/util/executeSchedule.ts", "node_modules/rxjs/src/internal/operators/observeOn.ts", "node_modules/rxjs/src/internal/operators/subscribeOn.ts", "node_modules/rxjs/src/internal/scheduled/scheduleObservable.ts", "node_modules/rxjs/src/internal/scheduled/schedulePromise.ts", "node_modules/rxjs/src/internal/scheduled/scheduleArray.ts", "node_modules/rxjs/src/internal/scheduled/scheduleIterable.ts", "node_modules/rxjs/src/internal/scheduled/scheduleAsyncIterable.ts", "node_modules/rxjs/src/internal/scheduled/scheduleReadableStreamLike.ts", "node_modules/rxjs/src/internal/scheduled/scheduled.ts", "node_modules/rxjs/src/internal/observable/from.ts", "node_modules/rxjs/src/internal/observable/of.ts", "node_modules/rxjs/src/internal/observable/throwError.ts", "node_modules/rxjs/src/internal/util/EmptyError.ts", "node_modules/rxjs/src/internal/util/isDate.ts", "node_modules/rxjs/src/internal/operators/map.ts", "node_modules/rxjs/src/internal/util/mapOneOrManyArgs.ts", "node_modules/rxjs/src/internal/util/argsArgArrayOrObject.ts", "node_modules/rxjs/src/internal/util/createObject.ts", "node_modules/rxjs/src/internal/observable/combineLatest.ts", "node_modules/rxjs/src/internal/operators/mergeInternals.ts", "node_modules/rxjs/src/internal/operators/mergeMap.ts", "node_modules/rxjs/src/internal/operators/mergeAll.ts", "node_modules/rxjs/src/internal/operators/concatAll.ts", "node_modules/rxjs/src/internal/observable/concat.ts", "node_modules/rxjs/src/internal/observable/defer.ts", "node_modules/rxjs/src/internal/observable/fromEvent.ts", "node_modules/rxjs/src/internal/observable/fromEventPattern.ts", "node_modules/rxjs/src/internal/observable/timer.ts", "node_modules/rxjs/src/internal/observable/merge.ts", "node_modules/rxjs/src/internal/observable/never.ts", "node_modules/rxjs/src/internal/util/argsOrArgArray.ts", "node_modules/rxjs/src/internal/operators/filter.ts", "node_modules/rxjs/src/internal/observable/zip.ts", "node_modules/rxjs/src/internal/operators/audit.ts", "node_modules/rxjs/src/internal/operators/auditTime.ts", "node_modules/rxjs/src/internal/operators/bufferCount.ts", "node_modules/rxjs/src/internal/operators/catchError.ts", "node_modules/rxjs/src/internal/operators/scanInternals.ts", "node_modules/rxjs/src/internal/operators/combineLatest.ts", "node_modules/rxjs/src/internal/operators/combineLatestWith.ts", "node_modules/rxjs/src/internal/operators/debounceTime.ts", "node_modules/rxjs/src/internal/operators/defaultIfEmpty.ts", "node_modules/rxjs/src/internal/operators/take.ts", "node_modules/rxjs/src/internal/operators/ignoreElements.ts", "node_modules/rxjs/src/internal/operators/mapTo.ts", "node_modules/rxjs/src/internal/operators/delayWhen.ts", "node_modules/rxjs/src/internal/operators/delay.ts", "node_modules/rxjs/src/internal/operators/distinctUntilChanged.ts", "node_modules/rxjs/src/internal/operators/distinctUntilKeyChanged.ts", "node_modules/rxjs/src/internal/operators/throwIfEmpty.ts", "node_modules/rxjs/src/internal/operators/endWith.ts", "node_modules/rxjs/src/internal/operators/finalize.ts", "node_modules/rxjs/src/internal/operators/first.ts", "node_modules/rxjs/src/internal/operators/takeLast.ts", "node_modules/rxjs/src/internal/operators/merge.ts", "node_modules/rxjs/src/internal/operators/mergeWith.ts", "node_modules/rxjs/src/internal/operators/repeat.ts", "node_modules/rxjs/src/internal/operators/scan.ts", "node_modules/rxjs/src/internal/operators/share.ts", "node_modules/rxjs/src/internal/operators/shareReplay.ts", "node_modules/rxjs/src/internal/operators/skip.ts", "node_modules/rxjs/src/internal/operators/skipUntil.ts", "node_modules/rxjs/src/internal/operators/startWith.ts", "node_modules/rxjs/src/internal/operators/switchMap.ts", "node_modules/rxjs/src/internal/operators/takeUntil.ts", "node_modules/rxjs/src/internal/operators/takeWhile.ts", "node_modules/rxjs/src/internal/operators/tap.ts", "node_modules/rxjs/src/internal/operators/throttle.ts", "node_modules/rxjs/src/internal/operators/throttleTime.ts", "node_modules/rxjs/src/internal/operators/withLatestFrom.ts", "node_modules/rxjs/src/internal/operators/zip.ts", "node_modules/rxjs/src/internal/operators/zipWith.ts", "src/templates/assets/javascripts/browser/document/index.ts", "src/templates/assets/javascripts/browser/element/_/index.ts", "src/templates/assets/javascripts/browser/element/focus/index.ts", "src/templates/assets/javascripts/browser/element/hover/index.ts", "src/templates/assets/javascripts/browser/element/offset/_/index.ts", "src/templates/assets/javascripts/browser/element/offset/content/index.ts", "src/templates/assets/javascripts/utilities/h/index.ts", "src/templates/assets/javascripts/utilities/round/index.ts", "src/templates/assets/javascripts/browser/script/index.ts", "src/templates/assets/javascripts/browser/element/size/_/index.ts", "src/templates/assets/javascripts/browser/element/size/content/index.ts", "src/templates/assets/javascripts/browser/element/visibility/index.ts", "src/templates/assets/javascripts/browser/toggle/index.ts", "src/templates/assets/javascripts/browser/keyboard/index.ts", "src/templates/assets/javascripts/browser/location/_/index.ts", "src/templates/assets/javascripts/browser/location/hash/index.ts", "src/templates/assets/javascripts/browser/media/index.ts", "src/templates/assets/javascripts/browser/request/index.ts", "src/templates/assets/javascripts/browser/viewport/offset/index.ts", "src/templates/assets/javascripts/browser/viewport/size/index.ts", "src/templates/assets/javascripts/browser/viewport/_/index.ts", "src/templates/assets/javascripts/browser/viewport/at/index.ts", "src/templates/assets/javascripts/browser/worker/index.ts", "src/templates/assets/javascripts/_/index.ts", "src/templates/assets/javascripts/components/_/index.ts", "src/templates/assets/javascripts/components/announce/index.ts", "src/templates/assets/javascripts/components/consent/index.ts", "src/templates/assets/javascripts/templates/tooltip/index.tsx", "src/templates/assets/javascripts/templates/annotation/index.tsx", "src/templates/assets/javascripts/templates/clipboard/index.tsx", "src/templates/assets/javascripts/templates/search/index.tsx", "src/templates/assets/javascripts/templates/source/index.tsx", "src/templates/assets/javascripts/templates/tabbed/index.tsx", "src/templates/assets/javascripts/templates/table/index.tsx", "src/templates/assets/javascripts/templates/version/index.tsx", "src/templates/assets/javascripts/components/tooltip/index.ts", "src/templates/assets/javascripts/components/content/annotation/_/index.ts", "src/templates/assets/javascripts/components/content/annotation/list/index.ts", "src/templates/assets/javascripts/components/content/annotation/block/index.ts", "src/templates/assets/javascripts/components/content/code/_/index.ts", "src/templates/assets/javascripts/components/content/details/index.ts", "src/templates/assets/javascripts/components/content/mermaid/index.css", "src/templates/assets/javascripts/components/content/mermaid/index.ts", "src/templates/assets/javascripts/components/content/table/index.ts", "src/templates/assets/javascripts/components/content/tabs/index.ts", "src/templates/assets/javascripts/components/content/_/index.ts", "src/templates/assets/javascripts/components/dialog/index.ts", "src/templates/assets/javascripts/components/header/_/index.ts", "src/templates/assets/javascripts/components/header/title/index.ts", "src/templates/assets/javascripts/components/main/index.ts", "src/templates/assets/javascripts/components/palette/index.ts", "src/templates/assets/javascripts/components/progress/index.ts", "src/templates/assets/javascripts/integrations/clipboard/index.ts", "src/templates/assets/javascripts/integrations/sitemap/index.ts", "src/templates/assets/javascripts/integrations/instant/index.ts", "src/templates/assets/javascripts/integrations/search/highlighter/index.ts", "src/templates/assets/javascripts/integrations/search/worker/message/index.ts", "src/templates/assets/javascripts/integrations/search/worker/_/index.ts", "src/templates/assets/javascripts/integrations/version/index.ts", "src/templates/assets/javascripts/components/search/query/index.ts", "src/templates/assets/javascripts/components/search/result/index.ts", "src/templates/assets/javascripts/components/search/share/index.ts", "src/templates/assets/javascripts/components/search/suggest/index.ts", "src/templates/assets/javascripts/components/search/_/index.ts", "src/templates/assets/javascripts/components/search/highlight/index.ts", "src/templates/assets/javascripts/components/sidebar/index.ts", "src/templates/assets/javascripts/components/source/facts/github/index.ts", "src/templates/assets/javascripts/components/source/facts/gitlab/index.ts", "src/templates/assets/javascripts/components/source/facts/_/index.ts", "src/templates/assets/javascripts/components/source/_/index.ts", "src/templates/assets/javascripts/components/tabs/index.ts", "src/templates/assets/javascripts/components/toc/index.ts", "src/templates/assets/javascripts/components/top/index.ts", "src/templates/assets/javascripts/patches/ellipsis/index.ts", "src/templates/assets/javascripts/patches/indeterminate/index.ts", "src/templates/assets/javascripts/patches/scrollfix/index.ts", "src/templates/assets/javascripts/patches/scrolllock/index.ts", "src/templates/assets/javascripts/polyfills/index.ts"],
+  "sourcesContent": ["(function (global, factory) {\n  typeof exports === 'object' && typeof module !== 'undefined' ? factory() :\n  typeof define === 'function' && define.amd ? define(factory) :\n  (factory());\n}(this, (function () { 'use strict';\n\n  /**\n   * Applies the :focus-visible polyfill at the given scope.\n   * A scope in this case is either the top-level Document or a Shadow Root.\n   *\n   * @param {(Document|ShadowRoot)} scope\n   * @see https://github.com/WICG/focus-visible\n   */\n  function applyFocusVisiblePolyfill(scope) {\n    var hadKeyboardEvent = true;\n    var hadFocusVisibleRecently = false;\n    var hadFocusVisibleRecentlyTimeout = null;\n\n    var inputTypesAllowlist = {\n      text: true,\n      search: true,\n      url: true,\n      tel: true,\n      email: true,\n      password: true,\n      number: true,\n      date: true,\n      month: true,\n      week: true,\n      time: true,\n      datetime: true,\n      'datetime-local': true\n    };\n\n    /**\n     * Helper function for legacy browsers and iframes which sometimes focus\n     * elements like document, body, and non-interactive SVG.\n     * @param {Element} el\n     */\n    function isValidFocusTarget(el) {\n      if (\n        el &&\n        el !== document &&\n        el.nodeName !== 'HTML' &&\n        el.nodeName !== 'BODY' &&\n        'classList' in el &&\n        'contains' in el.classList\n      ) {\n        return true;\n      }\n      return false;\n    }\n\n    /**\n     * Computes whether the given element should automatically trigger the\n     * `focus-visible` class being added, i.e. whether it should always match\n     * `:focus-visible` when focused.\n     * @param {Element} el\n     * @return {boolean}\n     */\n    function focusTriggersKeyboardModality(el) {\n      var type = el.type;\n      var tagName = el.tagName;\n\n      if (tagName === 'INPUT' && inputTypesAllowlist[type] && !el.readOnly) {\n        return true;\n      }\n\n      if (tagName === 'TEXTAREA' && !el.readOnly) {\n        return true;\n      }\n\n      if (el.isContentEditable) {\n        return true;\n      }\n\n      return false;\n    }\n\n    /**\n     * Add the `focus-visible` class to the given element if it was not added by\n     * the author.\n     * @param {Element} el\n     */\n    function addFocusVisibleClass(el) {\n      if (el.classList.contains('focus-visible')) {\n        return;\n      }\n      el.classList.add('focus-visible');\n      el.setAttribute('data-focus-visible-added', '');\n    }\n\n    /**\n     * Remove the `focus-visible` class from the given element if it was not\n     * originally added by the author.\n     * @param {Element} el\n     */\n    function removeFocusVisibleClass(el) {\n      if (!el.hasAttribute('data-focus-visible-added')) {\n        return;\n      }\n      el.classList.remove('focus-visible');\n      el.removeAttribute('data-focus-visible-added');\n    }\n\n    /**\n     * If the most recent user interaction was via the keyboard;\n     * and the key press did not include a meta, alt/option, or control key;\n     * then the modality is keyboard. Otherwise, the modality is not keyboard.\n     * Apply `focus-visible` to any current active element and keep track\n     * of our keyboard modality state with `hadKeyboardEvent`.\n     * @param {KeyboardEvent} e\n     */\n    function onKeyDown(e) {\n      if (e.metaKey || e.altKey || e.ctrlKey) {\n        return;\n      }\n\n      if (isValidFocusTarget(scope.activeElement)) {\n        addFocusVisibleClass(scope.activeElement);\n      }\n\n      hadKeyboardEvent = true;\n    }\n\n    /**\n     * If at any point a user clicks with a pointing device, ensure that we change\n     * the modality away from keyboard.\n     * This avoids the situation where a user presses a key on an already focused\n     * element, and then clicks on a different element, focusing it with a\n     * pointing device, while we still think we're in keyboard modality.\n     * @param {Event} e\n     */\n    function onPointerDown(e) {\n      hadKeyboardEvent = false;\n    }\n\n    /**\n     * On `focus`, add the `focus-visible` class to the target if:\n     * - the target received focus as a result of keyboard navigation, or\n     * - the event target is an element that will likely require interaction\n     *   via the keyboard (e.g. a text box)\n     * @param {Event} e\n     */\n    function onFocus(e) {\n      // Prevent IE from focusing the document or HTML element.\n      if (!isValidFocusTarget(e.target)) {\n        return;\n      }\n\n      if (hadKeyboardEvent || focusTriggersKeyboardModality(e.target)) {\n        addFocusVisibleClass(e.target);\n      }\n    }\n\n    /**\n     * On `blur`, remove the `focus-visible` class from the target.\n     * @param {Event} e\n     */\n    function onBlur(e) {\n      if (!isValidFocusTarget(e.target)) {\n        return;\n      }\n\n      if (\n        e.target.classList.contains('focus-visible') ||\n        e.target.hasAttribute('data-focus-visible-added')\n      ) {\n        // To detect a tab/window switch, we look for a blur event followed\n        // rapidly by a visibility change.\n        // If we don't see a visibility change within 100ms, it's probably a\n        // regular focus change.\n        hadFocusVisibleRecently = true;\n        window.clearTimeout(hadFocusVisibleRecentlyTimeout);\n        hadFocusVisibleRecentlyTimeout = window.setTimeout(function() {\n          hadFocusVisibleRecently = false;\n        }, 100);\n        removeFocusVisibleClass(e.target);\n      }\n    }\n\n    /**\n     * If the user changes tabs, keep track of whether or not the previously\n     * focused element had .focus-visible.\n     * @param {Event} e\n     */\n    function onVisibilityChange(e) {\n      if (document.visibilityState === 'hidden') {\n        // If the tab becomes active again, the browser will handle calling focus\n        // on the element (Safari actually calls it twice).\n        // If this tab change caused a blur on an element with focus-visible,\n        // re-apply the class when the user switches back to the tab.\n        if (hadFocusVisibleRecently) {\n          hadKeyboardEvent = true;\n        }\n        addInitialPointerMoveListeners();\n      }\n    }\n\n    /**\n     * Add a group of listeners to detect usage of any pointing devices.\n     * These listeners will be added when the polyfill first loads, and anytime\n     * the window is blurred, so that they are active when the window regains\n     * focus.\n     */\n    function addInitialPointerMoveListeners() {\n      document.addEventListener('mousemove', onInitialPointerMove);\n      document.addEventListener('mousedown', onInitialPointerMove);\n      document.addEventListener('mouseup', onInitialPointerMove);\n      document.addEventListener('pointermove', onInitialPointerMove);\n      document.addEventListener('pointerdown', onInitialPointerMove);\n      document.addEventListener('pointerup', onInitialPointerMove);\n      document.addEventListener('touchmove', onInitialPointerMove);\n      document.addEventListener('touchstart', onInitialPointerMove);\n      document.addEventListener('touchend', onInitialPointerMove);\n    }\n\n    function removeInitialPointerMoveListeners() {\n      document.removeEventListener('mousemove', onInitialPointerMove);\n      document.removeEventListener('mousedown', onInitialPointerMove);\n      document.removeEventListener('mouseup', onInitialPointerMove);\n      document.removeEventListener('pointermove', onInitialPointerMove);\n      document.removeEventListener('pointerdown', onInitialPointerMove);\n      document.removeEventListener('pointerup', onInitialPointerMove);\n      document.removeEventListener('touchmove', onInitialPointerMove);\n      document.removeEventListener('touchstart', onInitialPointerMove);\n      document.removeEventListener('touchend', onInitialPointerMove);\n    }\n\n    /**\n     * When the polfyill first loads, assume the user is in keyboard modality.\n     * If any event is received from a pointing device (e.g. mouse, pointer,\n     * touch), turn off keyboard modality.\n     * This accounts for situations where focus enters the page from the URL bar.\n     * @param {Event} e\n     */\n    function onInitialPointerMove(e) {\n      // Work around a Safari quirk that fires a mousemove on <html> whenever the\n      // window blurs, even if you're tabbing out of the page. \u00AF\\_(\u30C4)_/\u00AF\n      if (e.target.nodeName && e.target.nodeName.toLowerCase() === 'html') {\n        return;\n      }\n\n      hadKeyboardEvent = false;\n      removeInitialPointerMoveListeners();\n    }\n\n    // For some kinds of state, we are interested in changes at the global scope\n    // only. For example, global pointer input, global key presses and global\n    // visibility change should affect the state at every scope:\n    document.addEventListener('keydown', onKeyDown, true);\n    document.addEventListener('mousedown', onPointerDown, true);\n    document.addEventListener('pointerdown', onPointerDown, true);\n    document.addEventListener('touchstart', onPointerDown, true);\n    document.addEventListener('visibilitychange', onVisibilityChange, true);\n\n    addInitialPointerMoveListeners();\n\n    // For focus and blur, we specifically care about state changes in the local\n    // scope. This is because focus / blur events that originate from within a\n    // shadow root are not re-dispatched from the host element if it was already\n    // the active element in its own scope:\n    scope.addEventListener('focus', onFocus, true);\n    scope.addEventListener('blur', onBlur, true);\n\n    // We detect that a node is a ShadowRoot by ensuring that it is a\n    // DocumentFragment and also has a host property. This check covers native\n    // implementation and polyfill implementation transparently. If we only cared\n    // about the native implementation, we could just check if the scope was\n    // an instance of a ShadowRoot.\n    if (scope.nodeType === Node.DOCUMENT_FRAGMENT_NODE && scope.host) {\n      // Since a ShadowRoot is a special kind of DocumentFragment, it does not\n      // have a root element to add a class to. So, we add this attribute to the\n      // host element instead:\n      scope.host.setAttribute('data-js-focus-visible', '');\n    } else if (scope.nodeType === Node.DOCUMENT_NODE) {\n      document.documentElement.classList.add('js-focus-visible');\n      document.documentElement.setAttribute('data-js-focus-visible', '');\n    }\n  }\n\n  // It is important to wrap all references to global window and document in\n  // these checks to support server-side rendering use cases\n  // @see https://github.com/WICG/focus-visible/issues/199\n  if (typeof window !== 'undefined' && typeof document !== 'undefined') {\n    // Make the polyfill helper globally available. This can be used as a signal\n    // to interested libraries that wish to coordinate with the polyfill for e.g.,\n    // applying the polyfill to a shadow root:\n    window.applyFocusVisiblePolyfill = applyFocusVisiblePolyfill;\n\n    // Notify interested libraries of the polyfill's presence, in case the\n    // polyfill was loaded lazily:\n    var event;\n\n    try {\n      event = new CustomEvent('focus-visible-polyfill-ready');\n    } catch (error) {\n      // IE11 does not support using CustomEvent as a constructor directly:\n      event = document.createEvent('CustomEvent');\n      event.initCustomEvent('focus-visible-polyfill-ready', false, false, {});\n    }\n\n    window.dispatchEvent(event);\n  }\n\n  if (typeof document !== 'undefined') {\n    // Apply the polyfill to the global document, so that no JavaScript\n    // coordination is required to use the polyfill in the top-level document:\n    applyFocusVisiblePolyfill(document);\n  }\n\n})));\n", "/*!\n * clipboard.js v2.0.11\n * https://clipboardjs.com/\n *\n * Licensed MIT \u00A9 Zeno Rocha\n */\n(function webpackUniversalModuleDefinition(root, factory) {\n\tif(typeof exports === 'object' && typeof module === 'object')\n\t\tmodule.exports = factory();\n\telse if(typeof define === 'function' && define.amd)\n\t\tdefine([], factory);\n\telse if(typeof exports === 'object')\n\t\texports[\"ClipboardJS\"] = factory();\n\telse\n\t\troot[\"ClipboardJS\"] = factory();\n})(this, function() {\nreturn /******/ (function() { // webpackBootstrap\n/******/ \tvar __webpack_modules__ = ({\n\n/***/ 686:\n/***/ (function(__unused_webpack_module, __webpack_exports__, __webpack_require__) {\n\n\"use strict\";\n\n// EXPORTS\n__webpack_require__.d(__webpack_exports__, {\n  \"default\": function() { return /* binding */ clipboard; }\n});\n\n// EXTERNAL MODULE: ./node_modules/tiny-emitter/index.js\nvar tiny_emitter = __webpack_require__(279);\nvar tiny_emitter_default = /*#__PURE__*/__webpack_require__.n(tiny_emitter);\n// EXTERNAL MODULE: ./node_modules/good-listener/src/listen.js\nvar listen = __webpack_require__(370);\nvar listen_default = /*#__PURE__*/__webpack_require__.n(listen);\n// EXTERNAL MODULE: ./node_modules/select/src/select.js\nvar src_select = __webpack_require__(817);\nvar select_default = /*#__PURE__*/__webpack_require__.n(src_select);\n;// CONCATENATED MODULE: ./src/common/command.js\n/**\n * Executes a given operation type.\n * @param {String} type\n * @return {Boolean}\n */\nfunction command(type) {\n  try {\n    return document.execCommand(type);\n  } catch (err) {\n    return false;\n  }\n}\n;// CONCATENATED MODULE: ./src/actions/cut.js\n\n\n/**\n * Cut action wrapper.\n * @param {String|HTMLElement} target\n * @return {String}\n */\n\nvar ClipboardActionCut = function ClipboardActionCut(target) {\n  var selectedText = select_default()(target);\n  command('cut');\n  return selectedText;\n};\n\n/* harmony default export */ var actions_cut = (ClipboardActionCut);\n;// CONCATENATED MODULE: ./src/common/create-fake-element.js\n/**\n * Creates a fake textarea element with a value.\n * @param {String} value\n * @return {HTMLElement}\n */\nfunction createFakeElement(value) {\n  var isRTL = document.documentElement.getAttribute('dir') === 'rtl';\n  var fakeElement = document.createElement('textarea'); // Prevent zooming on iOS\n\n  fakeElement.style.fontSize = '12pt'; // Reset box model\n\n  fakeElement.style.border = '0';\n  fakeElement.style.padding = '0';\n  fakeElement.style.margin = '0'; // Move element out of screen horizontally\n\n  fakeElement.style.position = 'absolute';\n  fakeElement.style[isRTL ? 'right' : 'left'] = '-9999px'; // Move element to the same position vertically\n\n  var yPosition = window.pageYOffset || document.documentElement.scrollTop;\n  fakeElement.style.top = \"\".concat(yPosition, \"px\");\n  fakeElement.setAttribute('readonly', '');\n  fakeElement.value = value;\n  return fakeElement;\n}\n;// CONCATENATED MODULE: ./src/actions/copy.js\n\n\n\n/**\n * Create fake copy action wrapper using a fake element.\n * @param {String} target\n * @param {Object} options\n * @return {String}\n */\n\nvar fakeCopyAction = function fakeCopyAction(value, options) {\n  var fakeElement = createFakeElement(value);\n  options.container.appendChild(fakeElement);\n  var selectedText = select_default()(fakeElement);\n  command('copy');\n  fakeElement.remove();\n  return selectedText;\n};\n/**\n * Copy action wrapper.\n * @param {String|HTMLElement} target\n * @param {Object} options\n * @return {String}\n */\n\n\nvar ClipboardActionCopy = function ClipboardActionCopy(target) {\n  var options = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {\n    container: document.body\n  };\n  var selectedText = '';\n\n  if (typeof target === 'string') {\n    selectedText = fakeCopyAction(target, options);\n  } else if (target instanceof HTMLInputElement && !['text', 'search', 'url', 'tel', 'password'].includes(target === null || target === void 0 ? void 0 : target.type)) {\n    // If input type doesn't support `setSelectionRange`. Simulate it. https://developer.mozilla.org/en-US/docs/Web/API/HTMLInputElement/setSelectionRange\n    selectedText = fakeCopyAction(target.value, options);\n  } else {\n    selectedText = select_default()(target);\n    command('copy');\n  }\n\n  return selectedText;\n};\n\n/* harmony default export */ var actions_copy = (ClipboardActionCopy);\n;// CONCATENATED MODULE: ./src/actions/default.js\nfunction _typeof(obj) { \"@babel/helpers - typeof\"; if (typeof Symbol === \"function\" && typeof Symbol.iterator === \"symbol\") { _typeof = function _typeof(obj) { return typeof obj; }; } else { _typeof = function _typeof(obj) { return obj && typeof Symbol === \"function\" && obj.constructor === Symbol && obj !== Symbol.prototype ? \"symbol\" : typeof obj; }; } return _typeof(obj); }\n\n\n\n/**\n * Inner function which performs selection from either `text` or `target`\n * properties and then executes copy or cut operations.\n * @param {Object} options\n */\n\nvar ClipboardActionDefault = function ClipboardActionDefault() {\n  var options = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};\n  // Defines base properties passed from constructor.\n  var _options$action = options.action,\n      action = _options$action === void 0 ? 'copy' : _options$action,\n      container = options.container,\n      target = options.target,\n      text = options.text; // Sets the `action` to be performed which can be either 'copy' or 'cut'.\n\n  if (action !== 'copy' && action !== 'cut') {\n    throw new Error('Invalid \"action\" value, use either \"copy\" or \"cut\"');\n  } // Sets the `target` property using an element that will be have its content copied.\n\n\n  if (target !== undefined) {\n    if (target && _typeof(target) === 'object' && target.nodeType === 1) {\n      if (action === 'copy' && target.hasAttribute('disabled')) {\n        throw new Error('Invalid \"target\" attribute. Please use \"readonly\" instead of \"disabled\" attribute');\n      }\n\n      if (action === 'cut' && (target.hasAttribute('readonly') || target.hasAttribute('disabled'))) {\n        throw new Error('Invalid \"target\" attribute. You can\\'t cut text from elements with \"readonly\" or \"disabled\" attributes');\n      }\n    } else {\n      throw new Error('Invalid \"target\" value, use a valid Element');\n    }\n  } // Define selection strategy based on `text` property.\n\n\n  if (text) {\n    return actions_copy(text, {\n      container: container\n    });\n  } // Defines which selection strategy based on `target` property.\n\n\n  if (target) {\n    return action === 'cut' ? actions_cut(target) : actions_copy(target, {\n      container: container\n    });\n  }\n};\n\n/* harmony default export */ var actions_default = (ClipboardActionDefault);\n;// CONCATENATED MODULE: ./src/clipboard.js\nfunction clipboard_typeof(obj) { \"@babel/helpers - typeof\"; if (typeof Symbol === \"function\" && typeof Symbol.iterator === \"symbol\") { clipboard_typeof = function _typeof(obj) { return typeof obj; }; } else { clipboard_typeof = function _typeof(obj) { return obj && typeof Symbol === \"function\" && obj.constructor === Symbol && obj !== Symbol.prototype ? \"symbol\" : typeof obj; }; } return clipboard_typeof(obj); }\n\nfunction _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError(\"Cannot call a class as a function\"); } }\n\nfunction _defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if (\"value\" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } }\n\nfunction _createClass(Constructor, protoProps, staticProps) { if (protoProps) _defineProperties(Constructor.prototype, protoProps); if (staticProps) _defineProperties(Constructor, staticProps); return Constructor; }\n\nfunction _inherits(subClass, superClass) { if (typeof superClass !== \"function\" && superClass !== null) { throw new TypeError(\"Super expression must either be null or a function\"); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, writable: true, configurable: true } }); if (superClass) _setPrototypeOf(subClass, superClass); }\n\nfunction _setPrototypeOf(o, p) { _setPrototypeOf = Object.setPrototypeOf || function _setPrototypeOf(o, p) { o.__proto__ = p; return o; }; return _setPrototypeOf(o, p); }\n\nfunction _createSuper(Derived) { var hasNativeReflectConstruct = _isNativeReflectConstruct(); return function _createSuperInternal() { var Super = _getPrototypeOf(Derived), result; if (hasNativeReflectConstruct) { var NewTarget = _getPrototypeOf(this).constructor; result = Reflect.construct(Super, arguments, NewTarget); } else { result = Super.apply(this, arguments); } return _possibleConstructorReturn(this, result); }; }\n\nfunction _possibleConstructorReturn(self, call) { if (call && (clipboard_typeof(call) === \"object\" || typeof call === \"function\")) { return call; } return _assertThisInitialized(self); }\n\nfunction _assertThisInitialized(self) { if (self === void 0) { throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\"); } return self; }\n\nfunction _isNativeReflectConstruct() { if (typeof Reflect === \"undefined\" || !Reflect.construct) return false; if (Reflect.construct.sham) return false; if (typeof Proxy === \"function\") return true; try { Date.prototype.toString.call(Reflect.construct(Date, [], function () {})); return true; } catch (e) { return false; } }\n\nfunction _getPrototypeOf(o) { _getPrototypeOf = Object.setPrototypeOf ? Object.getPrototypeOf : function _getPrototypeOf(o) { return o.__proto__ || Object.getPrototypeOf(o); }; return _getPrototypeOf(o); }\n\n\n\n\n\n\n/**\n * Helper function to retrieve attribute value.\n * @param {String} suffix\n * @param {Element} element\n */\n\nfunction getAttributeValue(suffix, element) {\n  var attribute = \"data-clipboard-\".concat(suffix);\n\n  if (!element.hasAttribute(attribute)) {\n    return;\n  }\n\n  return element.getAttribute(attribute);\n}\n/**\n * Base class which takes one or more elements, adds event listeners to them,\n * and instantiates a new `ClipboardAction` on each click.\n */\n\n\nvar Clipboard = /*#__PURE__*/function (_Emitter) {\n  _inherits(Clipboard, _Emitter);\n\n  var _super = _createSuper(Clipboard);\n\n  /**\n   * @param {String|HTMLElement|HTMLCollection|NodeList} trigger\n   * @param {Object} options\n   */\n  function Clipboard(trigger, options) {\n    var _this;\n\n    _classCallCheck(this, Clipboard);\n\n    _this = _super.call(this);\n\n    _this.resolveOptions(options);\n\n    _this.listenClick(trigger);\n\n    return _this;\n  }\n  /**\n   * Defines if attributes would be resolved using internal setter functions\n   * or custom functions that were passed in the constructor.\n   * @param {Object} options\n   */\n\n\n  _createClass(Clipboard, [{\n    key: \"resolveOptions\",\n    value: function resolveOptions() {\n      var options = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};\n      this.action = typeof options.action === 'function' ? options.action : this.defaultAction;\n      this.target = typeof options.target === 'function' ? options.target : this.defaultTarget;\n      this.text = typeof options.text === 'function' ? options.text : this.defaultText;\n      this.container = clipboard_typeof(options.container) === 'object' ? options.container : document.body;\n    }\n    /**\n     * Adds a click event listener to the passed trigger.\n     * @param {String|HTMLElement|HTMLCollection|NodeList} trigger\n     */\n\n  }, {\n    key: \"listenClick\",\n    value: function listenClick(trigger) {\n      var _this2 = this;\n\n      this.listener = listen_default()(trigger, 'click', function (e) {\n        return _this2.onClick(e);\n      });\n    }\n    /**\n     * Defines a new `ClipboardAction` on each click event.\n     * @param {Event} e\n     */\n\n  }, {\n    key: \"onClick\",\n    value: function onClick(e) {\n      var trigger = e.delegateTarget || e.currentTarget;\n      var action = this.action(trigger) || 'copy';\n      var text = actions_default({\n        action: action,\n        container: this.container,\n        target: this.target(trigger),\n        text: this.text(trigger)\n      }); // Fires an event based on the copy operation result.\n\n      this.emit(text ? 'success' : 'error', {\n        action: action,\n        text: text,\n        trigger: trigger,\n        clearSelection: function clearSelection() {\n          if (trigger) {\n            trigger.focus();\n          }\n\n          window.getSelection().removeAllRanges();\n        }\n      });\n    }\n    /**\n     * Default `action` lookup function.\n     * @param {Element} trigger\n     */\n\n  }, {\n    key: \"defaultAction\",\n    value: function defaultAction(trigger) {\n      return getAttributeValue('action', trigger);\n    }\n    /**\n     * Default `target` lookup function.\n     * @param {Element} trigger\n     */\n\n  }, {\n    key: \"defaultTarget\",\n    value: function defaultTarget(trigger) {\n      var selector = getAttributeValue('target', trigger);\n\n      if (selector) {\n        return document.querySelector(selector);\n      }\n    }\n    /**\n     * Allow fire programmatically a copy action\n     * @param {String|HTMLElement} target\n     * @param {Object} options\n     * @returns Text copied.\n     */\n\n  }, {\n    key: \"defaultText\",\n\n    /**\n     * Default `text` lookup function.\n     * @param {Element} trigger\n     */\n    value: function defaultText(trigger) {\n      return getAttributeValue('text', trigger);\n    }\n    /**\n     * Destroy lifecycle.\n     */\n\n  }, {\n    key: \"destroy\",\n    value: function destroy() {\n      this.listener.destroy();\n    }\n  }], [{\n    key: \"copy\",\n    value: function copy(target) {\n      var options = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {\n        container: document.body\n      };\n      return actions_copy(target, options);\n    }\n    /**\n     * Allow fire programmatically a cut action\n     * @param {String|HTMLElement} target\n     * @returns Text cutted.\n     */\n\n  }, {\n    key: \"cut\",\n    value: function cut(target) {\n      return actions_cut(target);\n    }\n    /**\n     * Returns the support of the given action, or all actions if no action is\n     * given.\n     * @param {String} [action]\n     */\n\n  }, {\n    key: \"isSupported\",\n    value: function isSupported() {\n      var action = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : ['copy', 'cut'];\n      var actions = typeof action === 'string' ? [action] : action;\n      var support = !!document.queryCommandSupported;\n      actions.forEach(function (action) {\n        support = support && !!document.queryCommandSupported(action);\n      });\n      return support;\n    }\n  }]);\n\n  return Clipboard;\n}((tiny_emitter_default()));\n\n/* harmony default export */ var clipboard = (Clipboard);\n\n/***/ }),\n\n/***/ 828:\n/***/ (function(module) {\n\nvar DOCUMENT_NODE_TYPE = 9;\n\n/**\n * A polyfill for Element.matches()\n */\nif (typeof Element !== 'undefined' && !Element.prototype.matches) {\n    var proto = Element.prototype;\n\n    proto.matches = proto.matchesSelector ||\n                    proto.mozMatchesSelector ||\n                    proto.msMatchesSelector ||\n                    proto.oMatchesSelector ||\n                    proto.webkitMatchesSelector;\n}\n\n/**\n * Finds the closest parent that matches a selector.\n *\n * @param {Element} element\n * @param {String} selector\n * @return {Function}\n */\nfunction closest (element, selector) {\n    while (element && element.nodeType !== DOCUMENT_NODE_TYPE) {\n        if (typeof element.matches === 'function' &&\n            element.matches(selector)) {\n          return element;\n        }\n        element = element.parentNode;\n    }\n}\n\nmodule.exports = closest;\n\n\n/***/ }),\n\n/***/ 438:\n/***/ (function(module, __unused_webpack_exports, __webpack_require__) {\n\nvar closest = __webpack_require__(828);\n\n/**\n * Delegates event to a selector.\n *\n * @param {Element} element\n * @param {String} selector\n * @param {String} type\n * @param {Function} callback\n * @param {Boolean} useCapture\n * @return {Object}\n */\nfunction _delegate(element, selector, type, callback, useCapture) {\n    var listenerFn = listener.apply(this, arguments);\n\n    element.addEventListener(type, listenerFn, useCapture);\n\n    return {\n        destroy: function() {\n            element.removeEventListener(type, listenerFn, useCapture);\n        }\n    }\n}\n\n/**\n * Delegates event to a selector.\n *\n * @param {Element|String|Array} [elements]\n * @param {String} selector\n * @param {String} type\n * @param {Function} callback\n * @param {Boolean} useCapture\n * @return {Object}\n */\nfunction delegate(elements, selector, type, callback, useCapture) {\n    // Handle the regular Element usage\n    if (typeof elements.addEventListener === 'function') {\n        return _delegate.apply(null, arguments);\n    }\n\n    // Handle Element-less usage, it defaults to global delegation\n    if (typeof type === 'function') {\n        // Use `document` as the first parameter, then apply arguments\n        // This is a short way to .unshift `arguments` without running into deoptimizations\n        return _delegate.bind(null, document).apply(null, arguments);\n    }\n\n    // Handle Selector-based usage\n    if (typeof elements === 'string') {\n        elements = document.querySelectorAll(elements);\n    }\n\n    // Handle Array-like based usage\n    return Array.prototype.map.call(elements, function (element) {\n        return _delegate(element, selector, type, callback, useCapture);\n    });\n}\n\n/**\n * Finds closest match and invokes callback.\n *\n * @param {Element} element\n * @param {String} selector\n * @param {String} type\n * @param {Function} callback\n * @return {Function}\n */\nfunction listener(element, selector, type, callback) {\n    return function(e) {\n        e.delegateTarget = closest(e.target, selector);\n\n        if (e.delegateTarget) {\n            callback.call(element, e);\n        }\n    }\n}\n\nmodule.exports = delegate;\n\n\n/***/ }),\n\n/***/ 879:\n/***/ (function(__unused_webpack_module, exports) {\n\n/**\n * Check if argument is a HTML element.\n *\n * @param {Object} value\n * @return {Boolean}\n */\nexports.node = function(value) {\n    return value !== undefined\n        && value instanceof HTMLElement\n        && value.nodeType === 1;\n};\n\n/**\n * Check if argument is a list of HTML elements.\n *\n * @param {Object} value\n * @return {Boolean}\n */\nexports.nodeList = function(value) {\n    var type = Object.prototype.toString.call(value);\n\n    return value !== undefined\n        && (type === '[object NodeList]' || type === '[object HTMLCollection]')\n        && ('length' in value)\n        && (value.length === 0 || exports.node(value[0]));\n};\n\n/**\n * Check if argument is a string.\n *\n * @param {Object} value\n * @return {Boolean}\n */\nexports.string = function(value) {\n    return typeof value === 'string'\n        || value instanceof String;\n};\n\n/**\n * Check if argument is a function.\n *\n * @param {Object} value\n * @return {Boolean}\n */\nexports.fn = function(value) {\n    var type = Object.prototype.toString.call(value);\n\n    return type === '[object Function]';\n};\n\n\n/***/ }),\n\n/***/ 370:\n/***/ (function(module, __unused_webpack_exports, __webpack_require__) {\n\nvar is = __webpack_require__(879);\nvar delegate = __webpack_require__(438);\n\n/**\n * Validates all params and calls the right\n * listener function based on its target type.\n *\n * @param {String|HTMLElement|HTMLCollection|NodeList} target\n * @param {String} type\n * @param {Function} callback\n * @return {Object}\n */\nfunction listen(target, type, callback) {\n    if (!target && !type && !callback) {\n        throw new Error('Missing required arguments');\n    }\n\n    if (!is.string(type)) {\n        throw new TypeError('Second argument must be a String');\n    }\n\n    if (!is.fn(callback)) {\n        throw new TypeError('Third argument must be a Function');\n    }\n\n    if (is.node(target)) {\n        return listenNode(target, type, callback);\n    }\n    else if (is.nodeList(target)) {\n        return listenNodeList(target, type, callback);\n    }\n    else if (is.string(target)) {\n        return listenSelector(target, type, callback);\n    }\n    else {\n        throw new TypeError('First argument must be a String, HTMLElement, HTMLCollection, or NodeList');\n    }\n}\n\n/**\n * Adds an event listener to a HTML element\n * and returns a remove listener function.\n *\n * @param {HTMLElement} node\n * @param {String} type\n * @param {Function} callback\n * @return {Object}\n */\nfunction listenNode(node, type, callback) {\n    node.addEventListener(type, callback);\n\n    return {\n        destroy: function() {\n            node.removeEventListener(type, callback);\n        }\n    }\n}\n\n/**\n * Add an event listener to a list of HTML elements\n * and returns a remove listener function.\n *\n * @param {NodeList|HTMLCollection} nodeList\n * @param {String} type\n * @param {Function} callback\n * @return {Object}\n */\nfunction listenNodeList(nodeList, type, callback) {\n    Array.prototype.forEach.call(nodeList, function(node) {\n        node.addEventListener(type, callback);\n    });\n\n    return {\n        destroy: function() {\n            Array.prototype.forEach.call(nodeList, function(node) {\n                node.removeEventListener(type, callback);\n            });\n        }\n    }\n}\n\n/**\n * Add an event listener to a selector\n * and returns a remove listener function.\n *\n * @param {String} selector\n * @param {String} type\n * @param {Function} callback\n * @return {Object}\n */\nfunction listenSelector(selector, type, callback) {\n    return delegate(document.body, selector, type, callback);\n}\n\nmodule.exports = listen;\n\n\n/***/ }),\n\n/***/ 817:\n/***/ (function(module) {\n\nfunction select(element) {\n    var selectedText;\n\n    if (element.nodeName === 'SELECT') {\n        element.focus();\n\n        selectedText = element.value;\n    }\n    else if (element.nodeName === 'INPUT' || element.nodeName === 'TEXTAREA') {\n        var isReadOnly = element.hasAttribute('readonly');\n\n        if (!isReadOnly) {\n            element.setAttribute('readonly', '');\n        }\n\n        element.select();\n        element.setSelectionRange(0, element.value.length);\n\n        if (!isReadOnly) {\n            element.removeAttribute('readonly');\n        }\n\n        selectedText = element.value;\n    }\n    else {\n        if (element.hasAttribute('contenteditable')) {\n            element.focus();\n        }\n\n        var selection = window.getSelection();\n        var range = document.createRange();\n\n        range.selectNodeContents(element);\n        selection.removeAllRanges();\n        selection.addRange(range);\n\n        selectedText = selection.toString();\n    }\n\n    return selectedText;\n}\n\nmodule.exports = select;\n\n\n/***/ }),\n\n/***/ 279:\n/***/ (function(module) {\n\nfunction E () {\n  // Keep this empty so it's easier to inherit from\n  // (via https://github.com/lipsmack from https://github.com/scottcorgan/tiny-emitter/issues/3)\n}\n\nE.prototype = {\n  on: function (name, callback, ctx) {\n    var e = this.e || (this.e = {});\n\n    (e[name] || (e[name] = [])).push({\n      fn: callback,\n      ctx: ctx\n    });\n\n    return this;\n  },\n\n  once: function (name, callback, ctx) {\n    var self = this;\n    function listener () {\n      self.off(name, listener);\n      callback.apply(ctx, arguments);\n    };\n\n    listener._ = callback\n    return this.on(name, listener, ctx);\n  },\n\n  emit: function (name) {\n    var data = [].slice.call(arguments, 1);\n    var evtArr = ((this.e || (this.e = {}))[name] || []).slice();\n    var i = 0;\n    var len = evtArr.length;\n\n    for (i; i < len; i++) {\n      evtArr[i].fn.apply(evtArr[i].ctx, data);\n    }\n\n    return this;\n  },\n\n  off: function (name, callback) {\n    var e = this.e || (this.e = {});\n    var evts = e[name];\n    var liveEvents = [];\n\n    if (evts && callback) {\n      for (var i = 0, len = evts.length; i < len; i++) {\n        if (evts[i].fn !== callback && evts[i].fn._ !== callback)\n          liveEvents.push(evts[i]);\n      }\n    }\n\n    // Remove event from queue to prevent memory leak\n    // Suggested by https://github.com/lazd\n    // Ref: https://github.com/scottcorgan/tiny-emitter/commit/c6ebfaa9bc973b33d110a84a307742b7cf94c953#commitcomment-5024910\n\n    (liveEvents.length)\n      ? e[name] = liveEvents\n      : delete e[name];\n\n    return this;\n  }\n};\n\nmodule.exports = E;\nmodule.exports.TinyEmitter = E;\n\n\n/***/ })\n\n/******/ \t});\n/************************************************************************/\n/******/ \t// The module cache\n/******/ \tvar __webpack_module_cache__ = {};\n/******/ \t\n/******/ \t// The require function\n/******/ \tfunction __webpack_require__(moduleId) {\n/******/ \t\t// Check if module is in cache\n/******/ \t\tif(__webpack_module_cache__[moduleId]) {\n/******/ \t\t\treturn __webpack_module_cache__[moduleId].exports;\n/******/ \t\t}\n/******/ \t\t// Create a new module (and put it into the cache)\n/******/ \t\tvar module = __webpack_module_cache__[moduleId] = {\n/******/ \t\t\t// no module.id needed\n/******/ \t\t\t// no module.loaded needed\n/******/ \t\t\texports: {}\n/******/ \t\t};\n/******/ \t\n/******/ \t\t// Execute the module function\n/******/ \t\t__webpack_modules__[moduleId](module, module.exports, __webpack_require__);\n/******/ \t\n/******/ \t\t// Return the exports of the module\n/******/ \t\treturn module.exports;\n/******/ \t}\n/******/ \t\n/************************************************************************/\n/******/ \t/* webpack/runtime/compat get default export */\n/******/ \t!function() {\n/******/ \t\t// getDefaultExport function for compatibility with non-harmony modules\n/******/ \t\t__webpack_require__.n = function(module) {\n/******/ \t\t\tvar getter = module && module.__esModule ?\n/******/ \t\t\t\tfunction() { return module['default']; } :\n/******/ \t\t\t\tfunction() { return module; };\n/******/ \t\t\t__webpack_require__.d(getter, { a: getter });\n/******/ \t\t\treturn getter;\n/******/ \t\t};\n/******/ \t}();\n/******/ \t\n/******/ \t/* webpack/runtime/define property getters */\n/******/ \t!function() {\n/******/ \t\t// define getter functions for harmony exports\n/******/ \t\t__webpack_require__.d = function(exports, definition) {\n/******/ \t\t\tfor(var key in definition) {\n/******/ \t\t\t\tif(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n/******/ \t\t\t\t\tObject.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n/******/ \t\t\t\t}\n/******/ \t\t\t}\n/******/ \t\t};\n/******/ \t}();\n/******/ \t\n/******/ \t/* webpack/runtime/hasOwnProperty shorthand */\n/******/ \t!function() {\n/******/ \t\t__webpack_require__.o = function(obj, prop) { return Object.prototype.hasOwnProperty.call(obj, prop); }\n/******/ \t}();\n/******/ \t\n/************************************************************************/\n/******/ \t// module exports must be returned from runtime so entry inlining is disabled\n/******/ \t// startup\n/******/ \t// Load entry module and return exports\n/******/ \treturn __webpack_require__(686);\n/******/ })()\n.default;\n});", "/*!\n * escape-html\n * Copyright(c) 2012-2013 TJ Holowaychuk\n * Copyright(c) 2015 Andreas Lubbe\n * Copyright(c) 2015 Tiancheng \"Timothy\" Gu\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module variables.\n * @private\n */\n\nvar matchHtmlRegExp = /[\"'&<>]/;\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = escapeHtml;\n\n/**\n * Escape special characters in the given string of html.\n *\n * @param  {string} string The string to escape for inserting into HTML\n * @return {string}\n * @public\n */\n\nfunction escapeHtml(string) {\n  var str = '' + string;\n  var match = matchHtmlRegExp.exec(str);\n\n  if (!match) {\n    return str;\n  }\n\n  var escape;\n  var html = '';\n  var index = 0;\n  var lastIndex = 0;\n\n  for (index = match.index; index < str.length; index++) {\n    switch (str.charCodeAt(index)) {\n      case 34: // \"\n        escape = '&quot;';\n        break;\n      case 38: // &\n        escape = '&amp;';\n        break;\n      case 39: // '\n        escape = '&#39;';\n        break;\n      case 60: // <\n        escape = '&lt;';\n        break;\n      case 62: // >\n        escape = '&gt;';\n        break;\n      default:\n        continue;\n    }\n\n    if (lastIndex !== index) {\n      html += str.substring(lastIndex, index);\n    }\n\n    lastIndex = index + 1;\n    html += escape;\n  }\n\n  return lastIndex !== index\n    ? html + str.substring(lastIndex, index)\n    : html;\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport \"focus-visible\"\n\nimport {\n  EMPTY,\n  NEVER,\n  Observable,\n  Subject,\n  defer,\n  delay,\n  filter,\n  map,\n  merge,\n  mergeWith,\n  shareReplay,\n  switchMap\n} from \"rxjs\"\n\nimport { configuration, feature } from \"./_\"\nimport {\n  at,\n  getActiveElement,\n  getOptionalElement,\n  requestJSON,\n  setLocation,\n  setToggle,\n  watchDocument,\n  watchKeyboard,\n  watchLocation,\n  watchLocationTarget,\n  watchMedia,\n  watchPrint,\n  watchScript,\n  watchViewport\n} from \"./browser\"\nimport {\n  getComponentElement,\n  getComponentElements,\n  mountAnnounce,\n  mountBackToTop,\n  mountConsent,\n  mountContent,\n  mountDialog,\n  mountHeader,\n  mountHeaderTitle,\n  mountPalette,\n  mountProgress,\n  mountSearch,\n  mountSearchHiglight,\n  mountSidebar,\n  mountSource,\n  mountTableOfContents,\n  mountTabs,\n  watchHeader,\n  watchMain\n} from \"./components\"\nimport {\n  SearchIndex,\n  setupClipboardJS,\n  setupInstantNavigation,\n  setupVersionSelector\n} from \"./integrations\"\nimport {\n  patchEllipsis,\n  patchIndeterminate,\n  patchScrollfix,\n  patchScrolllock\n} from \"./patches\"\nimport \"./polyfills\"\n\n/* ----------------------------------------------------------------------------\n * Functions - @todo refactor\n * ------------------------------------------------------------------------- */\n\n/**\n * Fetch search index\n *\n * @returns Search index observable\n */\nfunction fetchSearchIndex(): Observable<SearchIndex> {\n  if (location.protocol === \"file:\") {\n    return watchScript(\n      `${new URL(\"search/search_index.js\", config.base)}`\n    )\n      .pipe(\n        // @ts-ignore - @todo fix typings\n        map(() => __index),\n        shareReplay(1)\n      )\n  } else {\n    return requestJSON<SearchIndex>(\n      new URL(\"search/search_index.json\", config.base)\n    )\n  }\n}\n\n/* ----------------------------------------------------------------------------\n * Application\n * ------------------------------------------------------------------------- */\n\n/* Yay, JavaScript is available */\ndocument.documentElement.classList.remove(\"no-js\")\ndocument.documentElement.classList.add(\"js\")\n\n/* Set up navigation observables and subjects */\nconst document$ = watchDocument()\nconst location$ = watchLocation()\nconst target$   = watchLocationTarget(location$)\nconst keyboard$ = watchKeyboard()\n\n/* Set up media observables */\nconst viewport$ = watchViewport()\nconst tablet$   = watchMedia(\"(min-width: 960px)\")\nconst screen$   = watchMedia(\"(min-width: 1220px)\")\nconst print$    = watchPrint()\n\n/* Retrieve search index, if search is enabled */\nconst config = configuration()\nconst index$ = document.forms.namedItem(\"search\")\n  ? fetchSearchIndex()\n  : NEVER\n\n/* Set up Clipboard.js integration */\nconst alert$ = new Subject<string>()\nsetupClipboardJS({ alert$ })\n\n/* Set up progress indicator */\nconst progress$ = new Subject<number>()\n\n/* Set up instant navigation, if enabled */\nif (feature(\"navigation.instant\"))\n  setupInstantNavigation({ location$, viewport$, progress$ })\n    .subscribe(document$)\n\n/* Set up version selector */\nif (config.version?.provider === \"mike\")\n  setupVersionSelector({ document$ })\n\n/* Always close drawer and search on navigation */\nmerge(location$, target$)\n  .pipe(\n    delay(125)\n  )\n    .subscribe(() => {\n      setToggle(\"drawer\", false)\n      setToggle(\"search\", false)\n    })\n\n/* Set up global keyboard handlers */\nkeyboard$\n  .pipe(\n    filter(({ mode }) => mode === \"global\")\n  )\n    .subscribe(key => {\n      switch (key.type) {\n\n        /* Go to previous page */\n        case \"p\":\n        case \",\":\n          const prev = getOptionalElement<HTMLLinkElement>(\"link[rel=prev]\")\n          if (typeof prev !== \"undefined\")\n            setLocation(prev)\n          break\n\n        /* Go to next page */\n        case \"n\":\n        case \".\":\n          const next = getOptionalElement<HTMLLinkElement>(\"link[rel=next]\")\n          if (typeof next !== \"undefined\")\n            setLocation(next)\n          break\n\n        /* Expand navigation, see https://bit.ly/3ZjG5io */\n        case \"Enter\":\n          const active = getActiveElement()\n          if (active instanceof HTMLLabelElement)\n            active.click()\n      }\n    })\n\n/* Set up patches */\npatchEllipsis({ document$ })\npatchIndeterminate({ document$, tablet$ })\npatchScrollfix({ document$ })\npatchScrolllock({ viewport$, tablet$ })\n\n/* Set up header and main area observable */\nconst header$ = watchHeader(getComponentElement(\"header\"), { viewport$ })\nconst main$ = document$\n  .pipe(\n    map(() => getComponentElement(\"main\")),\n    switchMap(el => watchMain(el, { viewport$, header$ })),\n    shareReplay(1)\n  )\n\n/* Set up control component observables */\nconst control$ = merge(\n\n  /* Consent */\n  ...getComponentElements(\"consent\")\n    .map(el => mountConsent(el, { target$ })),\n\n  /* Dialog */\n  ...getComponentElements(\"dialog\")\n    .map(el => mountDialog(el, { alert$ })),\n\n  /* Header */\n  ...getComponentElements(\"header\")\n    .map(el => mountHeader(el, { viewport$, header$, main$ })),\n\n  /* Color palette */\n  ...getComponentElements(\"palette\")\n    .map(el => mountPalette(el)),\n\n  /* Progress bar */\n  ...getComponentElements(\"progress\")\n    .map(el => mountProgress(el, { progress$ })),\n\n  /* Search */\n  ...getComponentElements(\"search\")\n    .map(el => mountSearch(el, { index$, keyboard$ })),\n\n  /* Repository information */\n  ...getComponentElements(\"source\")\n    .map(el => mountSource(el))\n)\n\n/* Set up content component observables */\nconst content$ = defer(() => merge(\n\n  /* Announcement bar */\n  ...getComponentElements(\"announce\")\n    .map(el => mountAnnounce(el)),\n\n  /* Content */\n  ...getComponentElements(\"content\")\n    .map(el => mountContent(el, { viewport$, target$, print$ })),\n\n  /* Search highlighting */\n  ...getComponentElements(\"content\")\n    .map(el => feature(\"search.highlight\")\n      ? mountSearchHiglight(el, { index$, location$ })\n      : EMPTY\n    ),\n\n  /* Header title */\n  ...getComponentElements(\"header-title\")\n    .map(el => mountHeaderTitle(el, { viewport$, header$ })),\n\n  /* Sidebar */\n  ...getComponentElements(\"sidebar\")\n    .map(el => el.getAttribute(\"data-md-type\") === \"navigation\"\n      ? at(screen$, () => mountSidebar(el, { viewport$, header$, main$ }))\n      : at(tablet$, () => mountSidebar(el, { viewport$, header$, main$ }))\n    ),\n\n  /* Navigation tabs */\n  ...getComponentElements(\"tabs\")\n    .map(el => mountTabs(el, { viewport$, header$ })),\n\n  /* Table of contents */\n  ...getComponentElements(\"toc\")\n    .map(el => mountTableOfContents(el, {\n      viewport$, header$, main$, target$\n    })),\n\n  /* Back-to-top button */\n  ...getComponentElements(\"top\")\n    .map(el => mountBackToTop(el, { viewport$, header$, main$, target$ }))\n))\n\n/* Set up component observables */\nconst component$ = document$\n  .pipe(\n    switchMap(() => content$),\n    mergeWith(control$),\n    shareReplay(1)\n  )\n\n/* Subscribe to all components */\ncomponent$.subscribe()\n\n/* ----------------------------------------------------------------------------\n * Exports\n * ------------------------------------------------------------------------- */\n\nwindow.document$  = document$          /* Document observable */\nwindow.location$  = location$          /* Location subject */\nwindow.target$    = target$            /* Location target observable */\nwindow.keyboard$  = keyboard$          /* Keyboard observable */\nwindow.viewport$  = viewport$          /* Viewport observable */\nwindow.tablet$    = tablet$            /* Media tablet observable */\nwindow.screen$    = screen$            /* Media screen observable */\nwindow.print$     = print$             /* Media print observable */\nwindow.alert$     = alert$             /* Alert subject */\nwindow.progress$  = progress$          /* Progress indicator subject */\nwindow.component$ = component$         /* Component observable */\n", "/*! *****************************************************************************\r\nCopyright (c) Microsoft Corporation.\r\n\r\nPermission to use, copy, modify, and/or distribute this software for any\r\npurpose with or without fee is hereby granted.\r\n\r\nTHE SOFTWARE IS PROVIDED \"AS IS\" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH\r\nREGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY\r\nAND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,\r\nINDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM\r\nLOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR\r\nOTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR\r\nPERFORMANCE OF THIS SOFTWARE.\r\n***************************************************************************** */\r\n/* global Reflect, Promise */\r\n\r\nvar extendStatics = function(d, b) {\r\n    extendStatics = Object.setPrototypeOf ||\r\n        ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||\r\n        function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };\r\n    return extendStatics(d, b);\r\n};\r\n\r\nexport function __extends(d, b) {\r\n    if (typeof b !== \"function\" && b !== null)\r\n        throw new TypeError(\"Class extends value \" + String(b) + \" is not a constructor or null\");\r\n    extendStatics(d, b);\r\n    function __() { this.constructor = d; }\r\n    d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());\r\n}\r\n\r\nexport var __assign = function() {\r\n    __assign = Object.assign || function __assign(t) {\r\n        for (var s, i = 1, n = arguments.length; i < n; i++) {\r\n            s = arguments[i];\r\n            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) t[p] = s[p];\r\n        }\r\n        return t;\r\n    }\r\n    return __assign.apply(this, arguments);\r\n}\r\n\r\nexport function __rest(s, e) {\r\n    var t = {};\r\n    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)\r\n        t[p] = s[p];\r\n    if (s != null && typeof Object.getOwnPropertySymbols === \"function\")\r\n        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {\r\n            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))\r\n                t[p[i]] = s[p[i]];\r\n        }\r\n    return t;\r\n}\r\n\r\nexport function __decorate(decorators, target, key, desc) {\r\n    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;\r\n    if (typeof Reflect === \"object\" && typeof Reflect.decorate === \"function\") r = Reflect.decorate(decorators, target, key, desc);\r\n    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;\r\n    return c > 3 && r && Object.defineProperty(target, key, r), r;\r\n}\r\n\r\nexport function __param(paramIndex, decorator) {\r\n    return function (target, key) { decorator(target, key, paramIndex); }\r\n}\r\n\r\nexport function __metadata(metadataKey, metadataValue) {\r\n    if (typeof Reflect === \"object\" && typeof Reflect.metadata === \"function\") return Reflect.metadata(metadataKey, metadataValue);\r\n}\r\n\r\nexport function __awaiter(thisArg, _arguments, P, generator) {\r\n    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }\r\n    return new (P || (P = Promise))(function (resolve, reject) {\r\n        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }\r\n        function rejected(value) { try { step(generator[\"throw\"](value)); } catch (e) { reject(e); } }\r\n        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }\r\n        step((generator = generator.apply(thisArg, _arguments || [])).next());\r\n    });\r\n}\r\n\r\nexport function __generator(thisArg, body) {\r\n    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;\r\n    return g = { next: verb(0), \"throw\": verb(1), \"return\": verb(2) }, typeof Symbol === \"function\" && (g[Symbol.iterator] = function() { return this; }), g;\r\n    function verb(n) { return function (v) { return step([n, v]); }; }\r\n    function step(op) {\r\n        if (f) throw new TypeError(\"Generator is already executing.\");\r\n        while (_) try {\r\n            if (f = 1, y && (t = op[0] & 2 ? y[\"return\"] : op[0] ? y[\"throw\"] || ((t = y[\"return\"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;\r\n            if (y = 0, t) op = [op[0] & 2, t.value];\r\n            switch (op[0]) {\r\n                case 0: case 1: t = op; break;\r\n                case 4: _.label++; return { value: op[1], done: false };\r\n                case 5: _.label++; y = op[1]; op = [0]; continue;\r\n                case 7: op = _.ops.pop(); _.trys.pop(); continue;\r\n                default:\r\n                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }\r\n                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }\r\n                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }\r\n                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }\r\n                    if (t[2]) _.ops.pop();\r\n                    _.trys.pop(); continue;\r\n            }\r\n            op = body.call(thisArg, _);\r\n        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }\r\n        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };\r\n    }\r\n}\r\n\r\nexport var __createBinding = Object.create ? (function(o, m, k, k2) {\r\n    if (k2 === undefined) k2 = k;\r\n    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });\r\n}) : (function(o, m, k, k2) {\r\n    if (k2 === undefined) k2 = k;\r\n    o[k2] = m[k];\r\n});\r\n\r\nexport function __exportStar(m, o) {\r\n    for (var p in m) if (p !== \"default\" && !Object.prototype.hasOwnProperty.call(o, p)) __createBinding(o, m, p);\r\n}\r\n\r\nexport function __values(o) {\r\n    var s = typeof Symbol === \"function\" && Symbol.iterator, m = s && o[s], i = 0;\r\n    if (m) return m.call(o);\r\n    if (o && typeof o.length === \"number\") return {\r\n        next: function () {\r\n            if (o && i >= o.length) o = void 0;\r\n            return { value: o && o[i++], done: !o };\r\n        }\r\n    };\r\n    throw new TypeError(s ? \"Object is not iterable.\" : \"Symbol.iterator is not defined.\");\r\n}\r\n\r\nexport function __read(o, n) {\r\n    var m = typeof Symbol === \"function\" && o[Symbol.iterator];\r\n    if (!m) return o;\r\n    var i = m.call(o), r, ar = [], e;\r\n    try {\r\n        while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);\r\n    }\r\n    catch (error) { e = { error: error }; }\r\n    finally {\r\n        try {\r\n            if (r && !r.done && (m = i[\"return\"])) m.call(i);\r\n        }\r\n        finally { if (e) throw e.error; }\r\n    }\r\n    return ar;\r\n}\r\n\r\n/** @deprecated */\r\nexport function __spread() {\r\n    for (var ar = [], i = 0; i < arguments.length; i++)\r\n        ar = ar.concat(__read(arguments[i]));\r\n    return ar;\r\n}\r\n\r\n/** @deprecated */\r\nexport function __spreadArrays() {\r\n    for (var s = 0, i = 0, il = arguments.length; i < il; i++) s += arguments[i].length;\r\n    for (var r = Array(s), k = 0, i = 0; i < il; i++)\r\n        for (var a = arguments[i], j = 0, jl = a.length; j < jl; j++, k++)\r\n            r[k] = a[j];\r\n    return r;\r\n}\r\n\r\nexport function __spreadArray(to, from, pack) {\r\n    if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {\r\n        if (ar || !(i in from)) {\r\n            if (!ar) ar = Array.prototype.slice.call(from, 0, i);\r\n            ar[i] = from[i];\r\n        }\r\n    }\r\n    return to.concat(ar || Array.prototype.slice.call(from));\r\n}\r\n\r\nexport function __await(v) {\r\n    return this instanceof __await ? (this.v = v, this) : new __await(v);\r\n}\r\n\r\nexport function __asyncGenerator(thisArg, _arguments, generator) {\r\n    if (!Symbol.asyncIterator) throw new TypeError(\"Symbol.asyncIterator is not defined.\");\r\n    var g = generator.apply(thisArg, _arguments || []), i, q = [];\r\n    return i = {}, verb(\"next\"), verb(\"throw\"), verb(\"return\"), i[Symbol.asyncIterator] = function () { return this; }, i;\r\n    function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; }\r\n    function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }\r\n    function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }\r\n    function fulfill(value) { resume(\"next\", value); }\r\n    function reject(value) { resume(\"throw\", value); }\r\n    function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }\r\n}\r\n\r\nexport function __asyncDelegator(o) {\r\n    var i, p;\r\n    return i = {}, verb(\"next\"), verb(\"throw\", function (e) { throw e; }), verb(\"return\"), i[Symbol.iterator] = function () { return this; }, i;\r\n    function verb(n, f) { i[n] = o[n] ? function (v) { return (p = !p) ? { value: __await(o[n](v)), done: n === \"return\" } : f ? f(v) : v; } : f; }\r\n}\r\n\r\nexport function __asyncValues(o) {\r\n    if (!Symbol.asyncIterator) throw new TypeError(\"Symbol.asyncIterator is not defined.\");\r\n    var m = o[Symbol.asyncIterator], i;\r\n    return m ? m.call(o) : (o = typeof __values === \"function\" ? __values(o) : o[Symbol.iterator](), i = {}, verb(\"next\"), verb(\"throw\"), verb(\"return\"), i[Symbol.asyncIterator] = function () { return this; }, i);\r\n    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }\r\n    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }\r\n}\r\n\r\nexport function __makeTemplateObject(cooked, raw) {\r\n    if (Object.defineProperty) { Object.defineProperty(cooked, \"raw\", { value: raw }); } else { cooked.raw = raw; }\r\n    return cooked;\r\n};\r\n\r\nvar __setModuleDefault = Object.create ? (function(o, v) {\r\n    Object.defineProperty(o, \"default\", { enumerable: true, value: v });\r\n}) : function(o, v) {\r\n    o[\"default\"] = v;\r\n};\r\n\r\nexport function __importStar(mod) {\r\n    if (mod && mod.__esModule) return mod;\r\n    var result = {};\r\n    if (mod != null) for (var k in mod) if (k !== \"default\" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);\r\n    __setModuleDefault(result, mod);\r\n    return result;\r\n}\r\n\r\nexport function __importDefault(mod) {\r\n    return (mod && mod.__esModule) ? mod : { default: mod };\r\n}\r\n\r\nexport function __classPrivateFieldGet(receiver, state, kind, f) {\r\n    if (kind === \"a\" && !f) throw new TypeError(\"Private accessor was defined without a getter\");\r\n    if (typeof state === \"function\" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError(\"Cannot read private member from an object whose class did not declare it\");\r\n    return kind === \"m\" ? f : kind === \"a\" ? f.call(receiver) : f ? f.value : state.get(receiver);\r\n}\r\n\r\nexport function __classPrivateFieldSet(receiver, state, value, kind, f) {\r\n    if (kind === \"m\") throw new TypeError(\"Private method is not writable\");\r\n    if (kind === \"a\" && !f) throw new TypeError(\"Private accessor was defined without a setter\");\r\n    if (typeof state === \"function\" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError(\"Cannot write private member to an object whose class did not declare it\");\r\n    return (kind === \"a\" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;\r\n}\r\n", "/**\n * Returns true if the object is a function.\n * @param value The value to check\n */\nexport function isFunction(value: any): value is (...args: any[]) => any {\n  return typeof value === 'function';\n}\n", "/**\n * Used to create Error subclasses until the community moves away from ES5.\n *\n * This is because compiling from TypeScript down to ES5 has issues with subclassing Errors\n * as well as other built-in types: https://github.com/Microsoft/TypeScript/issues/12123\n *\n * @param createImpl A factory function to create the actual constructor implementation. The returned\n * function should be a named function that calls `_super` internally.\n */\nexport function createErrorClass<T>(createImpl: (_super: any) => any): T {\n  const _super = (instance: any) => {\n    Error.call(instance);\n    instance.stack = new Error().stack;\n  };\n\n  const ctorFunc = createImpl(_super);\n  ctorFunc.prototype = Object.create(Error.prototype);\n  ctorFunc.prototype.constructor = ctorFunc;\n  return ctorFunc;\n}\n", "import { createErrorClass } from './createErrorClass';\n\nexport interface UnsubscriptionError extends Error {\n  readonly errors: any[];\n}\n\nexport interface UnsubscriptionErrorCtor {\n  /**\n   * @deprecated Internal implementation detail. Do not construct error instances.\n   * Cannot be tagged as internal: https://github.com/ReactiveX/rxjs/issues/6269\n   */\n  new (errors: any[]): UnsubscriptionError;\n}\n\n/**\n * An error thrown when one or more errors have occurred during the\n * `unsubscribe` of a {@link Subscription}.\n */\nexport const UnsubscriptionError: UnsubscriptionErrorCtor = createErrorClass(\n  (_super) =>\n    function UnsubscriptionErrorImpl(this: any, errors: (Error | string)[]) {\n      _super(this);\n      this.message = errors\n        ? `${errors.length} errors occurred during unsubscription:\n${errors.map((err, i) => `${i + 1}) ${err.toString()}`).join('\\n  ')}`\n        : '';\n      this.name = 'UnsubscriptionError';\n      this.errors = errors;\n    }\n);\n", "/**\n * Removes an item from an array, mutating it.\n * @param arr The array to remove the item from\n * @param item The item to remove\n */\nexport function arrRemove<T>(arr: T[] | undefined | null, item: T) {\n  if (arr) {\n    const index = arr.indexOf(item);\n    0 <= index && arr.splice(index, 1);\n  }\n}\n", "import { isFunction } from './util/isFunction';\nimport { UnsubscriptionError } from './util/UnsubscriptionError';\nimport { SubscriptionLike, TeardownLogic, Unsubscribable } from './types';\nimport { arrRemove } from './util/arrRemove';\n\n/**\n * Represents a disposable resource, such as the execution of an Observable. A\n * Subscription has one important method, `unsubscribe`, that takes no argument\n * and just disposes the resource held by the subscription.\n *\n * Additionally, subscriptions may be grouped together through the `add()`\n * method, which will attach a child Subscription to the current Subscription.\n * When a Subscription is unsubscribed, all its children (and its grandchildren)\n * will be unsubscribed as well.\n *\n * @class Subscription\n */\nexport class Subscription implements SubscriptionLike {\n  /** @nocollapse */\n  public static EMPTY = (() => {\n    const empty = new Subscription();\n    empty.closed = true;\n    return empty;\n  })();\n\n  /**\n   * A flag to indicate whether this Subscription has already been unsubscribed.\n   */\n  public closed = false;\n\n  private _parentage: Subscription[] | Subscription | null = null;\n\n  /**\n   * The list of registered finalizers to execute upon unsubscription. Adding and removing from this\n   * list occurs in the {@link #add} and {@link #remove} methods.\n   */\n  private _finalizers: Exclude<TeardownLogic, void>[] | null = null;\n\n  /**\n   * @param initialTeardown A function executed first as part of the finalization\n   * process that is kicked off when {@link #unsubscribe} is called.\n   */\n  constructor(private initialTeardown?: () => void) {}\n\n  /**\n   * Disposes the resources held by the subscription. May, for instance, cancel\n   * an ongoing Observable execution or cancel any other type of work that\n   * started when the Subscription was created.\n   * @return {void}\n   */\n  unsubscribe(): void {\n    let errors: any[] | undefined;\n\n    if (!this.closed) {\n      this.closed = true;\n\n      // Remove this from it's parents.\n      const { _parentage } = this;\n      if (_parentage) {\n        this._parentage = null;\n        if (Array.isArray(_parentage)) {\n          for (const parent of _parentage) {\n            parent.remove(this);\n          }\n        } else {\n          _parentage.remove(this);\n        }\n      }\n\n      const { initialTeardown: initialFinalizer } = this;\n      if (isFunction(initialFinalizer)) {\n        try {\n          initialFinalizer();\n        } catch (e) {\n          errors = e instanceof UnsubscriptionError ? e.errors : [e];\n        }\n      }\n\n      const { _finalizers } = this;\n      if (_finalizers) {\n        this._finalizers = null;\n        for (const finalizer of _finalizers) {\n          try {\n            execFinalizer(finalizer);\n          } catch (err) {\n            errors = errors ?? [];\n            if (err instanceof UnsubscriptionError) {\n              errors = [...errors, ...err.errors];\n            } else {\n              errors.push(err);\n            }\n          }\n        }\n      }\n\n      if (errors) {\n        throw new UnsubscriptionError(errors);\n      }\n    }\n  }\n\n  /**\n   * Adds a finalizer to this subscription, so that finalization will be unsubscribed/called\n   * when this subscription is unsubscribed. If this subscription is already {@link #closed},\n   * because it has already been unsubscribed, then whatever finalizer is passed to it\n   * will automatically be executed (unless the finalizer itself is also a closed subscription).\n   *\n   * Closed Subscriptions cannot be added as finalizers to any subscription. Adding a closed\n   * subscription to a any subscription will result in no operation. (A noop).\n   *\n   * Adding a subscription to itself, or adding `null` or `undefined` will not perform any\n   * operation at all. (A noop).\n   *\n   * `Subscription` instances that are added to this instance will automatically remove themselves\n   * if they are unsubscribed. Functions and {@link Unsubscribable} objects that you wish to remove\n   * will need to be removed manually with {@link #remove}\n   *\n   * @param teardown The finalization logic to add to this subscription.\n   */\n  add(teardown: TeardownLogic): void {\n    // Only add the finalizer if it's not undefined\n    // and don't add a subscription to itself.\n    if (teardown && teardown !== this) {\n      if (this.closed) {\n        // If this subscription is already closed,\n        // execute whatever finalizer is handed to it automatically.\n        execFinalizer(teardown);\n      } else {\n        if (teardown instanceof Subscription) {\n          // We don't add closed subscriptions, and we don't add the same subscription\n          // twice. Subscription unsubscribe is idempotent.\n          if (teardown.closed || teardown._hasParent(this)) {\n            return;\n          }\n          teardown._addParent(this);\n        }\n        (this._finalizers = this._finalizers ?? []).push(teardown);\n      }\n    }\n  }\n\n  /**\n   * Checks to see if a this subscription already has a particular parent.\n   * This will signal that this subscription has already been added to the parent in question.\n   * @param parent the parent to check for\n   */\n  private _hasParent(parent: Subscription) {\n    const { _parentage } = this;\n    return _parentage === parent || (Array.isArray(_parentage) && _parentage.includes(parent));\n  }\n\n  /**\n   * Adds a parent to this subscription so it can be removed from the parent if it\n   * unsubscribes on it's own.\n   *\n   * NOTE: THIS ASSUMES THAT {@link _hasParent} HAS ALREADY BEEN CHECKED.\n   * @param parent The parent subscription to add\n   */\n  private _addParent(parent: Subscription) {\n    const { _parentage } = this;\n    this._parentage = Array.isArray(_parentage) ? (_parentage.push(parent), _parentage) : _parentage ? [_parentage, parent] : parent;\n  }\n\n  /**\n   * Called on a child when it is removed via {@link #remove}.\n   * @param parent The parent to remove\n   */\n  private _removeParent(parent: Subscription) {\n    const { _parentage } = this;\n    if (_parentage === parent) {\n      this._parentage = null;\n    } else if (Array.isArray(_parentage)) {\n      arrRemove(_parentage, parent);\n    }\n  }\n\n  /**\n   * Removes a finalizer from this subscription that was previously added with the {@link #add} method.\n   *\n   * Note that `Subscription` instances, when unsubscribed, will automatically remove themselves\n   * from every other `Subscription` they have been added to. This means that using the `remove` method\n   * is not a common thing and should be used thoughtfully.\n   *\n   * If you add the same finalizer instance of a function or an unsubscribable object to a `Subscription` instance\n   * more than once, you will need to call `remove` the same number of times to remove all instances.\n   *\n   * All finalizer instances are removed to free up memory upon unsubscription.\n   *\n   * @param teardown The finalizer to remove from this subscription\n   */\n  remove(teardown: Exclude<TeardownLogic, void>): void {\n    const { _finalizers } = this;\n    _finalizers && arrRemove(_finalizers, teardown);\n\n    if (teardown instanceof Subscription) {\n      teardown._removeParent(this);\n    }\n  }\n}\n\nexport const EMPTY_SUBSCRIPTION = Subscription.EMPTY;\n\nexport function isSubscription(value: any): value is Subscription {\n  return (\n    value instanceof Subscription ||\n    (value && 'closed' in value && isFunction(value.remove) && isFunction(value.add) && isFunction(value.unsubscribe))\n  );\n}\n\nfunction execFinalizer(finalizer: Unsubscribable | (() => void)) {\n  if (isFunction(finalizer)) {\n    finalizer();\n  } else {\n    finalizer.unsubscribe();\n  }\n}\n", "import { Subscriber } from './Subscriber';\nimport { ObservableNotification } from './types';\n\n/**\n * The {@link GlobalConfig} object for RxJS. It is used to configure things\n * like how to react on unhandled errors.\n */\nexport const config: GlobalConfig = {\n  onUnhandledError: null,\n  onStoppedNotification: null,\n  Promise: undefined,\n  useDeprecatedSynchronousErrorHandling: false,\n  useDeprecatedNextContext: false,\n};\n\n/**\n * The global configuration object for RxJS, used to configure things\n * like how to react on unhandled errors. Accessible via {@link config}\n * object.\n */\nexport interface GlobalConfig {\n  /**\n   * A registration point for unhandled errors from RxJS. These are errors that\n   * cannot were not handled by consuming code in the usual subscription path. For\n   * example, if you have this configured, and you subscribe to an observable without\n   * providing an error handler, errors from that subscription will end up here. This\n   * will _always_ be called asynchronously on another job in the runtime. This is because\n   * we do not want errors thrown in this user-configured handler to interfere with the\n   * behavior of the library.\n   */\n  onUnhandledError: ((err: any) => void) | null;\n\n  /**\n   * A registration point for notifications that cannot be sent to subscribers because they\n   * have completed, errored or have been explicitly unsubscribed. By default, next, complete\n   * and error notifications sent to stopped subscribers are noops. However, sometimes callers\n   * might want a different behavior. For example, with sources that attempt to report errors\n   * to stopped subscribers, a caller can configure RxJS to throw an unhandled error instead.\n   * This will _always_ be called asynchronously on another job in the runtime. This is because\n   * we do not want errors thrown in this user-configured handler to interfere with the\n   * behavior of the library.\n   */\n  onStoppedNotification: ((notification: ObservableNotification<any>, subscriber: Subscriber<any>) => void) | null;\n\n  /**\n   * The promise constructor used by default for {@link Observable#toPromise toPromise} and {@link Observable#forEach forEach}\n   * methods.\n   *\n   * @deprecated As of version 8, RxJS will no longer support this sort of injection of a\n   * Promise constructor. If you need a Promise implementation other than native promises,\n   * please polyfill/patch Promise as you see appropriate. Will be removed in v8.\n   */\n  Promise?: PromiseConstructorLike;\n\n  /**\n   * If true, turns on synchronous error rethrowing, which is a deprecated behavior\n   * in v6 and higher. This behavior enables bad patterns like wrapping a subscribe\n   * call in a try/catch block. It also enables producer interference, a nasty bug\n   * where a multicast can be broken for all observers by a downstream consumer with\n   * an unhandled error. DO NOT USE THIS FLAG UNLESS IT'S NEEDED TO BUY TIME\n   * FOR MIGRATION REASONS.\n   *\n   * @deprecated As of version 8, RxJS will no longer support synchronous throwing\n   * of unhandled errors. All errors will be thrown on a separate call stack to prevent bad\n   * behaviors described above. Will be removed in v8.\n   */\n  useDeprecatedSynchronousErrorHandling: boolean;\n\n  /**\n   * If true, enables an as-of-yet undocumented feature from v5: The ability to access\n   * `unsubscribe()` via `this` context in `next` functions created in observers passed\n   * to `subscribe`.\n   *\n   * This is being removed because the performance was severely problematic, and it could also cause\n   * issues when types other than POJOs are passed to subscribe as subscribers, as they will likely have\n   * their `this` context overwritten.\n   *\n   * @deprecated As of version 8, RxJS will no longer support altering the\n   * context of next functions provided as part of an observer to Subscribe. Instead,\n   * you will have access to a subscription or a signal or token that will allow you to do things like\n   * unsubscribe and test closed status. Will be removed in v8.\n   */\n  useDeprecatedNextContext: boolean;\n}\n", "import type { TimerHandle } from './timerHandle';\ntype SetTimeoutFunction = (handler: () => void, timeout?: number, ...args: any[]) => TimerHandle;\ntype ClearTimeoutFunction = (handle: TimerHandle) => void;\n\ninterface TimeoutProvider {\n  setTimeout: SetTimeoutFunction;\n  clearTimeout: ClearTimeoutFunction;\n  delegate:\n    | {\n        setTimeout: SetTimeoutFunction;\n        clearTimeout: ClearTimeoutFunction;\n      }\n    | undefined;\n}\n\nexport const timeoutProvider: TimeoutProvider = {\n  // When accessing the delegate, use the variable rather than `this` so that\n  // the functions can be called without being bound to the provider.\n  setTimeout(handler: () => void, timeout?: number, ...args) {\n    const { delegate } = timeoutProvider;\n    if (delegate?.setTimeout) {\n      return delegate.setTimeout(handler, timeout, ...args);\n    }\n    return setTimeout(handler, timeout, ...args);\n  },\n  clearTimeout(handle) {\n    const { delegate } = timeoutProvider;\n    return (delegate?.clearTimeout || clearTimeout)(handle as any);\n  },\n  delegate: undefined,\n};\n", "import { config } from '../config';\nimport { timeoutProvider } from '../scheduler/timeoutProvider';\n\n/**\n * Handles an error on another job either with the user-configured {@link onUnhandledError},\n * or by throwing it on that new job so it can be picked up by `window.onerror`, `process.on('error')`, etc.\n *\n * This should be called whenever there is an error that is out-of-band with the subscription\n * or when an error hits a terminal boundary of the subscription and no error handler was provided.\n *\n * @param err the error to report\n */\nexport function reportUnhandledError(err: any) {\n  timeoutProvider.setTimeout(() => {\n    const { onUnhandledError } = config;\n    if (onUnhandledError) {\n      // Execute the user-configured error handler.\n      onUnhandledError(err);\n    } else {\n      // Throw so it is picked up by the runtime's uncaught error mechanism.\n      throw err;\n    }\n  });\n}\n", "/* tslint:disable:no-empty */\nexport function noop() { }\n", "import { CompleteNotification, NextNotification, ErrorNotification } from './types';\n\n/**\n * A completion object optimized for memory use and created to be the\n * same \"shape\" as other notifications in v8.\n * @internal\n */\nexport const COMPLETE_NOTIFICATION = (() => createNotification('C', undefined, undefined) as CompleteNotification)();\n\n/**\n * Internal use only. Creates an optimized error notification that is the same \"shape\"\n * as other notifications.\n * @internal\n */\nexport function errorNotification(error: any): ErrorNotification {\n  return createNotification('E', undefined, error) as any;\n}\n\n/**\n * Internal use only. Creates an optimized next notification that is the same \"shape\"\n * as other notifications.\n * @internal\n */\nexport function nextNotification<T>(value: T) {\n  return createNotification('N', value, undefined) as NextNotification<T>;\n}\n\n/**\n * Ensures that all notifications created internally have the same \"shape\" in v8.\n *\n * TODO: This is only exported to support a crazy legacy test in `groupBy`.\n * @internal\n */\nexport function createNotification(kind: 'N' | 'E' | 'C', value: any, error: any) {\n  return {\n    kind,\n    value,\n    error,\n  };\n}\n", "import { config } from '../config';\n\nlet context: { errorThrown: boolean; error: any } | null = null;\n\n/**\n * Handles dealing with errors for super-gross mode. Creates a context, in which\n * any synchronously thrown errors will be passed to {@link captureError}. Which\n * will record the error such that it will be rethrown after the call back is complete.\n * TODO: Remove in v8\n * @param cb An immediately executed function.\n */\nexport function errorContext(cb: () => void) {\n  if (config.useDeprecatedSynchronousErrorHandling) {\n    const isRoot = !context;\n    if (isRoot) {\n      context = { errorThrown: false, error: null };\n    }\n    cb();\n    if (isRoot) {\n      const { errorThrown, error } = context!;\n      context = null;\n      if (errorThrown) {\n        throw error;\n      }\n    }\n  } else {\n    // This is the general non-deprecated path for everyone that\n    // isn't crazy enough to use super-gross mode (useDeprecatedSynchronousErrorHandling)\n    cb();\n  }\n}\n\n/**\n * Captures errors only in super-gross mode.\n * @param err the error to capture\n */\nexport function captureError(err: any) {\n  if (config.useDeprecatedSynchronousErrorHandling && context) {\n    context.errorThrown = true;\n    context.error = err;\n  }\n}\n", "import { isFunction } from './util/isFunction';\nimport { Observer, ObservableNotification } from './types';\nimport { isSubscription, Subscription } from './Subscription';\nimport { config } from './config';\nimport { reportUnhandledError } from './util/reportUnhandledError';\nimport { noop } from './util/noop';\nimport { nextNotification, errorNotification, COMPLETE_NOTIFICATION } from './NotificationFactories';\nimport { timeoutProvider } from './scheduler/timeoutProvider';\nimport { captureError } from './util/errorContext';\n\n/**\n * Implements the {@link Observer} interface and extends the\n * {@link Subscription} class. While the {@link Observer} is the public API for\n * consuming the values of an {@link Observable}, all Observers get converted to\n * a Subscriber, in order to provide Subscription-like capabilities such as\n * `unsubscribe`. Subscriber is a common type in RxJS, and crucial for\n * implementing operators, but it is rarely used as a public API.\n *\n * @class Subscriber<T>\n */\nexport class Subscriber<T> extends Subscription implements Observer<T> {\n  /**\n   * A static factory for a Subscriber, given a (potentially partial) definition\n   * of an Observer.\n   * @param next The `next` callback of an Observer.\n   * @param error The `error` callback of an\n   * Observer.\n   * @param complete The `complete` callback of an\n   * Observer.\n   * @return A Subscriber wrapping the (partially defined)\n   * Observer represented by the given arguments.\n   * @nocollapse\n   * @deprecated Do not use. Will be removed in v8. There is no replacement for this\n   * method, and there is no reason to be creating instances of `Subscriber` directly.\n   * If you have a specific use case, please file an issue.\n   */\n  static create<T>(next?: (x?: T) => void, error?: (e?: any) => void, complete?: () => void): Subscriber<T> {\n    return new SafeSubscriber(next, error, complete);\n  }\n\n  /** @deprecated Internal implementation detail, do not use directly. Will be made internal in v8. */\n  protected isStopped: boolean = false;\n  /** @deprecated Internal implementation detail, do not use directly. Will be made internal in v8. */\n  protected destination: Subscriber<any> | Observer<any>; // this `any` is the escape hatch to erase extra type param (e.g. R)\n\n  /**\n   * @deprecated Internal implementation detail, do not use directly. Will be made internal in v8.\n   * There is no reason to directly create an instance of Subscriber. This type is exported for typings reasons.\n   */\n  constructor(destination?: Subscriber<any> | Observer<any>) {\n    super();\n    if (destination) {\n      this.destination = destination;\n      // Automatically chain subscriptions together here.\n      // if destination is a Subscription, then it is a Subscriber.\n      if (isSubscription(destination)) {\n        destination.add(this);\n      }\n    } else {\n      this.destination = EMPTY_OBSERVER;\n    }\n  }\n\n  /**\n   * The {@link Observer} callback to receive notifications of type `next` from\n   * the Observable, with a value. The Observable may call this method 0 or more\n   * times.\n   * @param {T} [value] The `next` value.\n   * @return {void}\n   */\n  next(value?: T): void {\n    if (this.isStopped) {\n      handleStoppedNotification(nextNotification(value), this);\n    } else {\n      this._next(value!);\n    }\n  }\n\n  /**\n   * The {@link Observer} callback to receive notifications of type `error` from\n   * the Observable, with an attached `Error`. Notifies the Observer that\n   * the Observable has experienced an error condition.\n   * @param {any} [err] The `error` exception.\n   * @return {void}\n   */\n  error(err?: any): void {\n    if (this.isStopped) {\n      handleStoppedNotification(errorNotification(err), this);\n    } else {\n      this.isStopped = true;\n      this._error(err);\n    }\n  }\n\n  /**\n   * The {@link Observer} callback to receive a valueless notification of type\n   * `complete` from the Observable. Notifies the Observer that the Observable\n   * has finished sending push-based notifications.\n   * @return {void}\n   */\n  complete(): void {\n    if (this.isStopped) {\n      handleStoppedNotification(COMPLETE_NOTIFICATION, this);\n    } else {\n      this.isStopped = true;\n      this._complete();\n    }\n  }\n\n  unsubscribe(): void {\n    if (!this.closed) {\n      this.isStopped = true;\n      super.unsubscribe();\n      this.destination = null!;\n    }\n  }\n\n  protected _next(value: T): void {\n    this.destination.next(value);\n  }\n\n  protected _error(err: any): void {\n    try {\n      this.destination.error(err);\n    } finally {\n      this.unsubscribe();\n    }\n  }\n\n  protected _complete(): void {\n    try {\n      this.destination.complete();\n    } finally {\n      this.unsubscribe();\n    }\n  }\n}\n\n/**\n * This bind is captured here because we want to be able to have\n * compatibility with monoid libraries that tend to use a method named\n * `bind`. In particular, a library called Monio requires this.\n */\nconst _bind = Function.prototype.bind;\n\nfunction bind<Fn extends (...args: any[]) => any>(fn: Fn, thisArg: any): Fn {\n  return _bind.call(fn, thisArg);\n}\n\n/**\n * Internal optimization only, DO NOT EXPOSE.\n * @internal\n */\nclass ConsumerObserver<T> implements Observer<T> {\n  constructor(private partialObserver: Partial<Observer<T>>) {}\n\n  next(value: T): void {\n    const { partialObserver } = this;\n    if (partialObserver.next) {\n      try {\n        partialObserver.next(value);\n      } catch (error) {\n        handleUnhandledError(error);\n      }\n    }\n  }\n\n  error(err: any): void {\n    const { partialObserver } = this;\n    if (partialObserver.error) {\n      try {\n        partialObserver.error(err);\n      } catch (error) {\n        handleUnhandledError(error);\n      }\n    } else {\n      handleUnhandledError(err);\n    }\n  }\n\n  complete(): void {\n    const { partialObserver } = this;\n    if (partialObserver.complete) {\n      try {\n        partialObserver.complete();\n      } catch (error) {\n        handleUnhandledError(error);\n      }\n    }\n  }\n}\n\nexport class SafeSubscriber<T> extends Subscriber<T> {\n  constructor(\n    observerOrNext?: Partial<Observer<T>> | ((value: T) => void) | null,\n    error?: ((e?: any) => void) | null,\n    complete?: (() => void) | null\n  ) {\n    super();\n\n    let partialObserver: Partial<Observer<T>>;\n    if (isFunction(observerOrNext) || !observerOrNext) {\n      // The first argument is a function, not an observer. The next\n      // two arguments *could* be observers, or they could be empty.\n      partialObserver = {\n        next: (observerOrNext ?? undefined) as (((value: T) => void) | undefined),\n        error: error ?? undefined,\n        complete: complete ?? undefined,\n      };\n    } else {\n      // The first argument is a partial observer.\n      let context: any;\n      if (this && config.useDeprecatedNextContext) {\n        // This is a deprecated path that made `this.unsubscribe()` available in\n        // next handler functions passed to subscribe. This only exists behind a flag\n        // now, as it is *very* slow.\n        context = Object.create(observerOrNext);\n        context.unsubscribe = () => this.unsubscribe();\n        partialObserver = {\n          next: observerOrNext.next && bind(observerOrNext.next, context),\n          error: observerOrNext.error && bind(observerOrNext.error, context),\n          complete: observerOrNext.complete && bind(observerOrNext.complete, context),\n        };\n      } else {\n        // The \"normal\" path. Just use the partial observer directly.\n        partialObserver = observerOrNext;\n      }\n    }\n\n    // Wrap the partial observer to ensure it's a full observer, and\n    // make sure proper error handling is accounted for.\n    this.destination = new ConsumerObserver(partialObserver);\n  }\n}\n\nfunction handleUnhandledError(error: any) {\n  if (config.useDeprecatedSynchronousErrorHandling) {\n    captureError(error);\n  } else {\n    // Ideal path, we report this as an unhandled error,\n    // which is thrown on a new call stack.\n    reportUnhandledError(error);\n  }\n}\n\n/**\n * An error handler used when no error handler was supplied\n * to the SafeSubscriber -- meaning no error handler was supplied\n * do the `subscribe` call on our observable.\n * @param err The error to handle\n */\nfunction defaultErrorHandler(err: any) {\n  throw err;\n}\n\n/**\n * A handler for notifications that cannot be sent to a stopped subscriber.\n * @param notification The notification being sent\n * @param subscriber The stopped subscriber\n */\nfunction handleStoppedNotification(notification: ObservableNotification<any>, subscriber: Subscriber<any>) {\n  const { onStoppedNotification } = config;\n  onStoppedNotification && timeoutProvider.setTimeout(() => onStoppedNotification(notification, subscriber));\n}\n\n/**\n * The observer used as a stub for subscriptions where the user did not\n * pass any arguments to `subscribe`. Comes with the default error handling\n * behavior.\n */\nexport const EMPTY_OBSERVER: Readonly<Observer<any>> & { closed: true } = {\n  closed: true,\n  next: noop,\n  error: defaultErrorHandler,\n  complete: noop,\n};\n", "/**\n * Symbol.observable or a string \"@@observable\". Used for interop\n *\n * @deprecated We will no longer be exporting this symbol in upcoming versions of RxJS.\n * Instead polyfill and use Symbol.observable directly *or* use https://www.npmjs.com/package/symbol-observable\n */\nexport const observable: string | symbol = (() => (typeof Symbol === 'function' && Symbol.observable) || '@@observable')();\n", "/**\n * This function takes one parameter and just returns it. Simply put,\n * this is like `<T>(x: T): T => x`.\n *\n * ## Examples\n *\n * This is useful in some cases when using things like `mergeMap`\n *\n * ```ts\n * import { interval, take, map, range, mergeMap, identity } from 'rxjs';\n *\n * const source$ = interval(1000).pipe(take(5));\n *\n * const result$ = source$.pipe(\n *   map(i => range(i)),\n *   mergeMap(identity) // same as mergeMap(x => x)\n * );\n *\n * result$.subscribe({\n *   next: console.log\n * });\n * ```\n *\n * Or when you want to selectively apply an operator\n *\n * ```ts\n * import { interval, take, identity } from 'rxjs';\n *\n * const shouldLimit = () => Math.random() < 0.5;\n *\n * const source$ = interval(1000);\n *\n * const result$ = source$.pipe(shouldLimit() ? take(5) : identity);\n *\n * result$.subscribe({\n *   next: console.log\n * });\n * ```\n *\n * @param x Any value that is returned by this function\n * @returns The value passed as the first parameter to this function\n */\nexport function identity<T>(x: T): T {\n  return x;\n}\n", "import { identity } from './identity';\nimport { UnaryFunction } from '../types';\n\nexport function pipe(): typeof identity;\nexport function pipe<T, A>(fn1: UnaryFunction<T, A>): UnaryFunction<T, A>;\nexport function pipe<T, A, B>(fn1: UnaryFunction<T, A>, fn2: UnaryFunction<A, B>): UnaryFunction<T, B>;\nexport function pipe<T, A, B, C>(fn1: UnaryFunction<T, A>, fn2: UnaryFunction<A, B>, fn3: UnaryFunction<B, C>): UnaryFunction<T, C>;\nexport function pipe<T, A, B, C, D>(\n  fn1: UnaryFunction<T, A>,\n  fn2: UnaryFunction<A, B>,\n  fn3: UnaryFunction<B, C>,\n  fn4: UnaryFunction<C, D>\n): UnaryFunction<T, D>;\nexport function pipe<T, A, B, C, D, E>(\n  fn1: UnaryFunction<T, A>,\n  fn2: UnaryFunction<A, B>,\n  fn3: UnaryFunction<B, C>,\n  fn4: UnaryFunction<C, D>,\n  fn5: UnaryFunction<D, E>\n): UnaryFunction<T, E>;\nexport function pipe<T, A, B, C, D, E, F>(\n  fn1: UnaryFunction<T, A>,\n  fn2: UnaryFunction<A, B>,\n  fn3: UnaryFunction<B, C>,\n  fn4: UnaryFunction<C, D>,\n  fn5: UnaryFunction<D, E>,\n  fn6: UnaryFunction<E, F>\n): UnaryFunction<T, F>;\nexport function pipe<T, A, B, C, D, E, F, G>(\n  fn1: UnaryFunction<T, A>,\n  fn2: UnaryFunction<A, B>,\n  fn3: UnaryFunction<B, C>,\n  fn4: UnaryFunction<C, D>,\n  fn5: UnaryFunction<D, E>,\n  fn6: UnaryFunction<E, F>,\n  fn7: UnaryFunction<F, G>\n): UnaryFunction<T, G>;\nexport function pipe<T, A, B, C, D, E, F, G, H>(\n  fn1: UnaryFunction<T, A>,\n  fn2: UnaryFunction<A, B>,\n  fn3: UnaryFunction<B, C>,\n  fn4: UnaryFunction<C, D>,\n  fn5: UnaryFunction<D, E>,\n  fn6: UnaryFunction<E, F>,\n  fn7: UnaryFunction<F, G>,\n  fn8: UnaryFunction<G, H>\n): UnaryFunction<T, H>;\nexport function pipe<T, A, B, C, D, E, F, G, H, I>(\n  fn1: UnaryFunction<T, A>,\n  fn2: UnaryFunction<A, B>,\n  fn3: UnaryFunction<B, C>,\n  fn4: UnaryFunction<C, D>,\n  fn5: UnaryFunction<D, E>,\n  fn6: UnaryFunction<E, F>,\n  fn7: UnaryFunction<F, G>,\n  fn8: UnaryFunction<G, H>,\n  fn9: UnaryFunction<H, I>\n): UnaryFunction<T, I>;\nexport function pipe<T, A, B, C, D, E, F, G, H, I>(\n  fn1: UnaryFunction<T, A>,\n  fn2: UnaryFunction<A, B>,\n  fn3: UnaryFunction<B, C>,\n  fn4: UnaryFunction<C, D>,\n  fn5: UnaryFunction<D, E>,\n  fn6: UnaryFunction<E, F>,\n  fn7: UnaryFunction<F, G>,\n  fn8: UnaryFunction<G, H>,\n  fn9: UnaryFunction<H, I>,\n  ...fns: UnaryFunction<any, any>[]\n): UnaryFunction<T, unknown>;\n\n/**\n * pipe() can be called on one or more functions, each of which can take one argument (\"UnaryFunction\")\n * and uses it to return a value.\n * It returns a function that takes one argument, passes it to the first UnaryFunction, and then\n * passes the result to the next one, passes that result to the next one, and so on.  \n */\nexport function pipe(...fns: Array<UnaryFunction<any, any>>): UnaryFunction<any, any> {\n  return pipeFromArray(fns);\n}\n\n/** @internal */\nexport function pipeFromArray<T, R>(fns: Array<UnaryFunction<T, R>>): UnaryFunction<T, R> {\n  if (fns.length === 0) {\n    return identity as UnaryFunction<any, any>;\n  }\n\n  if (fns.length === 1) {\n    return fns[0];\n  }\n\n  return function piped(input: T): R {\n    return fns.reduce((prev: any, fn: UnaryFunction<T, R>) => fn(prev), input as any);\n  };\n}\n", "import { Operator } from './Operator';\nimport { SafeSubscriber, Subscriber } from './Subscriber';\nimport { isSubscription, Subscription } from './Subscription';\nimport { TeardownLogic, OperatorFunction, Subscribable, Observer } from './types';\nimport { observable as Symbol_observable } from './symbol/observable';\nimport { pipeFromArray } from './util/pipe';\nimport { config } from './config';\nimport { isFunction } from './util/isFunction';\nimport { errorContext } from './util/errorContext';\n\n/**\n * A representation of any set of values over any amount of time. This is the most basic building block\n * of RxJS.\n *\n * @class Observable<T>\n */\nexport class Observable<T> implements Subscribable<T> {\n  /**\n   * @deprecated Internal implementation detail, do not use directly. Will be made internal in v8.\n   */\n  source: Observable<any> | undefined;\n\n  /**\n   * @deprecated Internal implementation detail, do not use directly. Will be made internal in v8.\n   */\n  operator: Operator<any, T> | undefined;\n\n  /**\n   * @constructor\n   * @param {Function} subscribe the function that is called when the Observable is\n   * initially subscribed to. This function is given a Subscriber, to which new values\n   * can be `next`ed, or an `error` method can be called to raise an error, or\n   * `complete` can be called to notify of a successful completion.\n   */\n  constructor(subscribe?: (this: Observable<T>, subscriber: Subscriber<T>) => TeardownLogic) {\n    if (subscribe) {\n      this._subscribe = subscribe;\n    }\n  }\n\n  // HACK: Since TypeScript inherits static properties too, we have to\n  // fight against TypeScript here so Subject can have a different static create signature\n  /**\n   * Creates a new Observable by calling the Observable constructor\n   * @owner Observable\n   * @method create\n   * @param {Function} subscribe? the subscriber function to be passed to the Observable constructor\n   * @return {Observable} a new observable\n   * @nocollapse\n   * @deprecated Use `new Observable()` instead. Will be removed in v8.\n   */\n  static create: (...args: any[]) => any = <T>(subscribe?: (subscriber: Subscriber<T>) => TeardownLogic) => {\n    return new Observable<T>(subscribe);\n  };\n\n  /**\n   * Creates a new Observable, with this Observable instance as the source, and the passed\n   * operator defined as the new observable's operator.\n   * @method lift\n   * @param operator the operator defining the operation to take on the observable\n   * @return a new observable with the Operator applied\n   * @deprecated Internal implementation detail, do not use directly. Will be made internal in v8.\n   * If you have implemented an operator using `lift`, it is recommended that you create an\n   * operator by simply returning `new Observable()` directly. See \"Creating new operators from\n   * scratch\" section here: https://rxjs.dev/guide/operators\n   */\n  lift<R>(operator?: Operator<T, R>): Observable<R> {\n    const observable = new Observable<R>();\n    observable.source = this;\n    observable.operator = operator;\n    return observable;\n  }\n\n  subscribe(observerOrNext?: Partial<Observer<T>> | ((value: T) => void)): Subscription;\n  /** @deprecated Instead of passing separate callback arguments, use an observer argument. Signatures taking separate callback arguments will be removed in v8. Details: https://rxjs.dev/deprecations/subscribe-arguments */\n  subscribe(next?: ((value: T) => void) | null, error?: ((error: any) => void) | null, complete?: (() => void) | null): Subscription;\n  /**\n   * Invokes an execution of an Observable and registers Observer handlers for notifications it will emit.\n   *\n   * <span class=\"informal\">Use it when you have all these Observables, but still nothing is happening.</span>\n   *\n   * `subscribe` is not a regular operator, but a method that calls Observable's internal `subscribe` function. It\n   * might be for example a function that you passed to Observable's constructor, but most of the time it is\n   * a library implementation, which defines what will be emitted by an Observable, and when it be will emitted. This means\n   * that calling `subscribe` is actually the moment when Observable starts its work, not when it is created, as it is often\n   * the thought.\n   *\n   * Apart from starting the execution of an Observable, this method allows you to listen for values\n   * that an Observable emits, as well as for when it completes or errors. You can achieve this in two\n   * of the following ways.\n   *\n   * The first way is creating an object that implements {@link Observer} interface. It should have methods\n   * defined by that interface, but note that it should be just a regular JavaScript object, which you can create\n   * yourself in any way you want (ES6 class, classic function constructor, object literal etc.). In particular, do\n   * not attempt to use any RxJS implementation details to create Observers - you don't need them. Remember also\n   * that your object does not have to implement all methods. If you find yourself creating a method that doesn't\n   * do anything, you can simply omit it. Note however, if the `error` method is not provided and an error happens,\n   * it will be thrown asynchronously. Errors thrown asynchronously cannot be caught using `try`/`catch`. Instead,\n   * use the {@link onUnhandledError} configuration option or use a runtime handler (like `window.onerror` or\n   * `process.on('error)`) to be notified of unhandled errors. Because of this, it's recommended that you provide\n   * an `error` method to avoid missing thrown errors.\n   *\n   * The second way is to give up on Observer object altogether and simply provide callback functions in place of its methods.\n   * This means you can provide three functions as arguments to `subscribe`, where the first function is equivalent\n   * of a `next` method, the second of an `error` method and the third of a `complete` method. Just as in case of an Observer,\n   * if you do not need to listen for something, you can omit a function by passing `undefined` or `null`,\n   * since `subscribe` recognizes these functions by where they were placed in function call. When it comes\n   * to the `error` function, as with an Observer, if not provided, errors emitted by an Observable will be thrown asynchronously.\n   *\n   * You can, however, subscribe with no parameters at all. This may be the case where you're not interested in terminal events\n   * and you also handled emissions internally by using operators (e.g. using `tap`).\n   *\n   * Whichever style of calling `subscribe` you use, in both cases it returns a Subscription object.\n   * This object allows you to call `unsubscribe` on it, which in turn will stop the work that an Observable does and will clean\n   * up all resources that an Observable used. Note that cancelling a subscription will not call `complete` callback\n   * provided to `subscribe` function, which is reserved for a regular completion signal that comes from an Observable.\n   *\n   * Remember that callbacks provided to `subscribe` are not guaranteed to be called asynchronously.\n   * It is an Observable itself that decides when these functions will be called. For example {@link of}\n   * by default emits all its values synchronously. Always check documentation for how given Observable\n   * will behave when subscribed and if its default behavior can be modified with a `scheduler`.\n   *\n   * #### Examples\n   *\n   * Subscribe with an {@link guide/observer Observer}\n   *\n   * ```ts\n   * import { of } from 'rxjs';\n   *\n   * const sumObserver = {\n   *   sum: 0,\n   *   next(value) {\n   *     console.log('Adding: ' + value);\n   *     this.sum = this.sum + value;\n   *   },\n   *   error() {\n   *     // We actually could just remove this method,\n   *     // since we do not really care about errors right now.\n   *   },\n   *   complete() {\n   *     console.log('Sum equals: ' + this.sum);\n   *   }\n   * };\n   *\n   * of(1, 2, 3) // Synchronously emits 1, 2, 3 and then completes.\n   *   .subscribe(sumObserver);\n   *\n   * // Logs:\n   * // 'Adding: 1'\n   * // 'Adding: 2'\n   * // 'Adding: 3'\n   * // 'Sum equals: 6'\n   * ```\n   *\n   * Subscribe with functions ({@link deprecations/subscribe-arguments deprecated})\n   *\n   * ```ts\n   * import { of } from 'rxjs'\n   *\n   * let sum = 0;\n   *\n   * of(1, 2, 3).subscribe(\n   *   value => {\n   *     console.log('Adding: ' + value);\n   *     sum = sum + value;\n   *   },\n   *   undefined,\n   *   () => console.log('Sum equals: ' + sum)\n   * );\n   *\n   * // Logs:\n   * // 'Adding: 1'\n   * // 'Adding: 2'\n   * // 'Adding: 3'\n   * // 'Sum equals: 6'\n   * ```\n   *\n   * Cancel a subscription\n   *\n   * ```ts\n   * import { interval } from 'rxjs';\n   *\n   * const subscription = interval(1000).subscribe({\n   *   next(num) {\n   *     console.log(num)\n   *   },\n   *   complete() {\n   *     // Will not be called, even when cancelling subscription.\n   *     console.log('completed!');\n   *   }\n   * });\n   *\n   * setTimeout(() => {\n   *   subscription.unsubscribe();\n   *   console.log('unsubscribed!');\n   * }, 2500);\n   *\n   * // Logs:\n   * // 0 after 1s\n   * // 1 after 2s\n   * // 'unsubscribed!' after 2.5s\n   * ```\n   *\n   * @param {Observer|Function} observerOrNext (optional) Either an observer with methods to be called,\n   * or the first of three possible handlers, which is the handler for each value emitted from the subscribed\n   * Observable.\n   * @param {Function} error (optional) A handler for a terminal event resulting from an error. If no error handler is provided,\n   * the error will be thrown asynchronously as unhandled.\n   * @param {Function} complete (optional) A handler for a terminal event resulting from successful completion.\n   * @return {Subscription} a subscription reference to the registered handlers\n   * @method subscribe\n   */\n  subscribe(\n    observerOrNext?: Partial<Observer<T>> | ((value: T) => void) | null,\n    error?: ((error: any) => void) | null,\n    complete?: (() => void) | null\n  ): Subscription {\n    const subscriber = isSubscriber(observerOrNext) ? observerOrNext : new SafeSubscriber(observerOrNext, error, complete);\n\n    errorContext(() => {\n      const { operator, source } = this;\n      subscriber.add(\n        operator\n          ? // We're dealing with a subscription in the\n            // operator chain to one of our lifted operators.\n            operator.call(subscriber, source)\n          : source\n          ? // If `source` has a value, but `operator` does not, something that\n            // had intimate knowledge of our API, like our `Subject`, must have\n            // set it. We're going to just call `_subscribe` directly.\n            this._subscribe(subscriber)\n          : // In all other cases, we're likely wrapping a user-provided initializer\n            // function, so we need to catch errors and handle them appropriately.\n            this._trySubscribe(subscriber)\n      );\n    });\n\n    return subscriber;\n  }\n\n  /** @internal */\n  protected _trySubscribe(sink: Subscriber<T>): TeardownLogic {\n    try {\n      return this._subscribe(sink);\n    } catch (err) {\n      // We don't need to return anything in this case,\n      // because it's just going to try to `add()` to a subscription\n      // above.\n      sink.error(err);\n    }\n  }\n\n  /**\n   * Used as a NON-CANCELLABLE means of subscribing to an observable, for use with\n   * APIs that expect promises, like `async/await`. You cannot unsubscribe from this.\n   *\n   * **WARNING**: Only use this with observables you *know* will complete. If the source\n   * observable does not complete, you will end up with a promise that is hung up, and\n   * potentially all of the state of an async function hanging out in memory. To avoid\n   * this situation, look into adding something like {@link timeout}, {@link take},\n   * {@link takeWhile}, or {@link takeUntil} amongst others.\n   *\n   * #### Example\n   *\n   * ```ts\n   * import { interval, take } from 'rxjs';\n   *\n   * const source$ = interval(1000).pipe(take(4));\n   *\n   * async function getTotal() {\n   *   let total = 0;\n   *\n   *   await source$.forEach(value => {\n   *     total += value;\n   *     console.log('observable -> ' + value);\n   *   });\n   *\n   *   return total;\n   * }\n   *\n   * getTotal().then(\n   *   total => console.log('Total: ' + total)\n   * );\n   *\n   * // Expected:\n   * // 'observable -> 0'\n   * // 'observable -> 1'\n   * // 'observable -> 2'\n   * // 'observable -> 3'\n   * // 'Total: 6'\n   * ```\n   *\n   * @param next a handler for each value emitted by the observable\n   * @return a promise that either resolves on observable completion or\n   *  rejects with the handled error\n   */\n  forEach(next: (value: T) => void): Promise<void>;\n\n  /**\n   * @param next a handler for each value emitted by the observable\n   * @param promiseCtor a constructor function used to instantiate the Promise\n   * @return a promise that either resolves on observable completion or\n   *  rejects with the handled error\n   * @deprecated Passing a Promise constructor will no longer be available\n   * in upcoming versions of RxJS. This is because it adds weight to the library, for very\n   * little benefit. If you need this functionality, it is recommended that you either\n   * polyfill Promise, or you create an adapter to convert the returned native promise\n   * to whatever promise implementation you wanted. Will be removed in v8.\n   */\n  forEach(next: (value: T) => void, promiseCtor: PromiseConstructorLike): Promise<void>;\n\n  forEach(next: (value: T) => void, promiseCtor?: PromiseConstructorLike): Promise<void> {\n    promiseCtor = getPromiseCtor(promiseCtor);\n\n    return new promiseCtor<void>((resolve, reject) => {\n      const subscriber = new SafeSubscriber<T>({\n        next: (value) => {\n          try {\n            next(value);\n          } catch (err) {\n            reject(err);\n            subscriber.unsubscribe();\n          }\n        },\n        error: reject,\n        complete: resolve,\n      });\n      this.subscribe(subscriber);\n    }) as Promise<void>;\n  }\n\n  /** @internal */\n  protected _subscribe(subscriber: Subscriber<any>): TeardownLogic {\n    return this.source?.subscribe(subscriber);\n  }\n\n  /**\n   * An interop point defined by the es7-observable spec https://github.com/zenparsing/es-observable\n   * @method Symbol.observable\n   * @return {Observable} this instance of the observable\n   */\n  [Symbol_observable]() {\n    return this;\n  }\n\n  /* tslint:disable:max-line-length */\n  pipe(): Observable<T>;\n  pipe<A>(op1: OperatorFunction<T, A>): Observable<A>;\n  pipe<A, B>(op1: OperatorFunction<T, A>, op2: OperatorFunction<A, B>): Observable<B>;\n  pipe<A, B, C>(op1: OperatorFunction<T, A>, op2: OperatorFunction<A, B>, op3: OperatorFunction<B, C>): Observable<C>;\n  pipe<A, B, C, D>(\n    op1: OperatorFunction<T, A>,\n    op2: OperatorFunction<A, B>,\n    op3: OperatorFunction<B, C>,\n    op4: OperatorFunction<C, D>\n  ): Observable<D>;\n  pipe<A, B, C, D, E>(\n    op1: OperatorFunction<T, A>,\n    op2: OperatorFunction<A, B>,\n    op3: OperatorFunction<B, C>,\n    op4: OperatorFunction<C, D>,\n    op5: OperatorFunction<D, E>\n  ): Observable<E>;\n  pipe<A, B, C, D, E, F>(\n    op1: OperatorFunction<T, A>,\n    op2: OperatorFunction<A, B>,\n    op3: OperatorFunction<B, C>,\n    op4: OperatorFunction<C, D>,\n    op5: OperatorFunction<D, E>,\n    op6: OperatorFunction<E, F>\n  ): Observable<F>;\n  pipe<A, B, C, D, E, F, G>(\n    op1: OperatorFunction<T, A>,\n    op2: OperatorFunction<A, B>,\n    op3: OperatorFunction<B, C>,\n    op4: OperatorFunction<C, D>,\n    op5: OperatorFunction<D, E>,\n    op6: OperatorFunction<E, F>,\n    op7: OperatorFunction<F, G>\n  ): Observable<G>;\n  pipe<A, B, C, D, E, F, G, H>(\n    op1: OperatorFunction<T, A>,\n    op2: OperatorFunction<A, B>,\n    op3: OperatorFunction<B, C>,\n    op4: OperatorFunction<C, D>,\n    op5: OperatorFunction<D, E>,\n    op6: OperatorFunction<E, F>,\n    op7: OperatorFunction<F, G>,\n    op8: OperatorFunction<G, H>\n  ): Observable<H>;\n  pipe<A, B, C, D, E, F, G, H, I>(\n    op1: OperatorFunction<T, A>,\n    op2: OperatorFunction<A, B>,\n    op3: OperatorFunction<B, C>,\n    op4: OperatorFunction<C, D>,\n    op5: OperatorFunction<D, E>,\n    op6: OperatorFunction<E, F>,\n    op7: OperatorFunction<F, G>,\n    op8: OperatorFunction<G, H>,\n    op9: OperatorFunction<H, I>\n  ): Observable<I>;\n  pipe<A, B, C, D, E, F, G, H, I>(\n    op1: OperatorFunction<T, A>,\n    op2: OperatorFunction<A, B>,\n    op3: OperatorFunction<B, C>,\n    op4: OperatorFunction<C, D>,\n    op5: OperatorFunction<D, E>,\n    op6: OperatorFunction<E, F>,\n    op7: OperatorFunction<F, G>,\n    op8: OperatorFunction<G, H>,\n    op9: OperatorFunction<H, I>,\n    ...operations: OperatorFunction<any, any>[]\n  ): Observable<unknown>;\n  /* tslint:enable:max-line-length */\n\n  /**\n   * Used to stitch together functional operators into a chain.\n   * @method pipe\n   * @return {Observable} the Observable result of all of the operators having\n   * been called in the order they were passed in.\n   *\n   * ## Example\n   *\n   * ```ts\n   * import { interval, filter, map, scan } from 'rxjs';\n   *\n   * interval(1000)\n   *   .pipe(\n   *     filter(x => x % 2 === 0),\n   *     map(x => x + x),\n   *     scan((acc, x) => acc + x)\n   *   )\n   *   .subscribe(x => console.log(x));\n   * ```\n   */\n  pipe(...operations: OperatorFunction<any, any>[]): Observable<any> {\n    return pipeFromArray(operations)(this);\n  }\n\n  /* tslint:disable:max-line-length */\n  /** @deprecated Replaced with {@link firstValueFrom} and {@link lastValueFrom}. Will be removed in v8. Details: https://rxjs.dev/deprecations/to-promise */\n  toPromise(): Promise<T | undefined>;\n  /** @deprecated Replaced with {@link firstValueFrom} and {@link lastValueFrom}. Will be removed in v8. Details: https://rxjs.dev/deprecations/to-promise */\n  toPromise(PromiseCtor: typeof Promise): Promise<T | undefined>;\n  /** @deprecated Replaced with {@link firstValueFrom} and {@link lastValueFrom}. Will be removed in v8. Details: https://rxjs.dev/deprecations/to-promise */\n  toPromise(PromiseCtor: PromiseConstructorLike): Promise<T | undefined>;\n  /* tslint:enable:max-line-length */\n\n  /**\n   * Subscribe to this Observable and get a Promise resolving on\n   * `complete` with the last emission (if any).\n   *\n   * **WARNING**: Only use this with observables you *know* will complete. If the source\n   * observable does not complete, you will end up with a promise that is hung up, and\n   * potentially all of the state of an async function hanging out in memory. To avoid\n   * this situation, look into adding something like {@link timeout}, {@link take},\n   * {@link takeWhile}, or {@link takeUntil} amongst others.\n   *\n   * @method toPromise\n   * @param [promiseCtor] a constructor function used to instantiate\n   * the Promise\n   * @return A Promise that resolves with the last value emit, or\n   * rejects on an error. If there were no emissions, Promise\n   * resolves with undefined.\n   * @deprecated Replaced with {@link firstValueFrom} and {@link lastValueFrom}. Will be removed in v8. Details: https://rxjs.dev/deprecations/to-promise\n   */\n  toPromise(promiseCtor?: PromiseConstructorLike): Promise<T | undefined> {\n    promiseCtor = getPromiseCtor(promiseCtor);\n\n    return new promiseCtor((resolve, reject) => {\n      let value: T | undefined;\n      this.subscribe(\n        (x: T) => (value = x),\n        (err: any) => reject(err),\n        () => resolve(value)\n      );\n    }) as Promise<T | undefined>;\n  }\n}\n\n/**\n * Decides between a passed promise constructor from consuming code,\n * A default configured promise constructor, and the native promise\n * constructor and returns it. If nothing can be found, it will throw\n * an error.\n * @param promiseCtor The optional promise constructor to passed by consuming code\n */\nfunction getPromiseCtor(promiseCtor: PromiseConstructorLike | undefined) {\n  return promiseCtor ?? config.Promise ?? Promise;\n}\n\nfunction isObserver<T>(value: any): value is Observer<T> {\n  return value && isFunction(value.next) && isFunction(value.error) && isFunction(value.complete);\n}\n\nfunction isSubscriber<T>(value: any): value is Subscriber<T> {\n  return (value && value instanceof Subscriber) || (isObserver(value) && isSubscription(value));\n}\n", "import { Observable } from '../Observable';\nimport { Subscriber } from '../Subscriber';\nimport { OperatorFunction } from '../types';\nimport { isFunction } from './isFunction';\n\n/**\n * Used to determine if an object is an Observable with a lift function.\n */\nexport function hasLift(source: any): source is { lift: InstanceType<typeof Observable>['lift'] } {\n  return isFunction(source?.lift);\n}\n\n/**\n * Creates an `OperatorFunction`. Used to define operators throughout the library in a concise way.\n * @param init The logic to connect the liftedSource to the subscriber at the moment of subscription.\n */\nexport function operate<T, R>(\n  init: (liftedSource: Observable<T>, subscriber: Subscriber<R>) => (() => void) | void\n): OperatorFunction<T, R> {\n  return (source: Observable<T>) => {\n    if (hasLift(source)) {\n      return source.lift(function (this: Subscriber<R>, liftedSource: Observable<T>) {\n        try {\n          return init(liftedSource, this);\n        } catch (err) {\n          this.error(err);\n        }\n      });\n    }\n    throw new TypeError('Unable to lift unknown Observable type');\n  };\n}\n", "import { Subscriber } from '../Subscriber';\n\n/**\n * Creates an instance of an `OperatorSubscriber`.\n * @param destination The downstream subscriber.\n * @param onNext Handles next values, only called if this subscriber is not stopped or closed. Any\n * error that occurs in this function is caught and sent to the `error` method of this subscriber.\n * @param onError Handles errors from the subscription, any errors that occur in this handler are caught\n * and send to the `destination` error handler.\n * @param onComplete Handles completion notification from the subscription. Any errors that occur in\n * this handler are sent to the `destination` error handler.\n * @param onFinalize Additional teardown logic here. This will only be called on teardown if the\n * subscriber itself is not already closed. This is called after all other teardown logic is executed.\n */\nexport function createOperatorSubscriber<T>(\n  destination: Subscriber<any>,\n  onNext?: (value: T) => void,\n  onComplete?: () => void,\n  onError?: (err: any) => void,\n  onFinalize?: () => void\n): Subscriber<T> {\n  return new OperatorSubscriber(destination, onNext, onComplete, onError, onFinalize);\n}\n\n/**\n * A generic helper for allowing operators to be created with a Subscriber and\n * use closures to capture necessary state from the operator function itself.\n */\nexport class OperatorSubscriber<T> extends Subscriber<T> {\n  /**\n   * Creates an instance of an `OperatorSubscriber`.\n   * @param destination The downstream subscriber.\n   * @param onNext Handles next values, only called if this subscriber is not stopped or closed. Any\n   * error that occurs in this function is caught and sent to the `error` method of this subscriber.\n   * @param onError Handles errors from the subscription, any errors that occur in this handler are caught\n   * and send to the `destination` error handler.\n   * @param onComplete Handles completion notification from the subscription. Any errors that occur in\n   * this handler are sent to the `destination` error handler.\n   * @param onFinalize Additional finalization logic here. This will only be called on finalization if the\n   * subscriber itself is not already closed. This is called after all other finalization logic is executed.\n   * @param shouldUnsubscribe An optional check to see if an unsubscribe call should truly unsubscribe.\n   * NOTE: This currently **ONLY** exists to support the strange behavior of {@link groupBy}, where unsubscription\n   * to the resulting observable does not actually disconnect from the source if there are active subscriptions\n   * to any grouped observable. (DO NOT EXPOSE OR USE EXTERNALLY!!!)\n   */\n  constructor(\n    destination: Subscriber<any>,\n    onNext?: (value: T) => void,\n    onComplete?: () => void,\n    onError?: (err: any) => void,\n    private onFinalize?: () => void,\n    private shouldUnsubscribe?: () => boolean\n  ) {\n    // It's important - for performance reasons - that all of this class's\n    // members are initialized and that they are always initialized in the same\n    // order. This will ensure that all OperatorSubscriber instances have the\n    // same hidden class in V8. This, in turn, will help keep the number of\n    // hidden classes involved in property accesses within the base class as\n    // low as possible. If the number of hidden classes involved exceeds four,\n    // the property accesses will become megamorphic and performance penalties\n    // will be incurred - i.e. inline caches won't be used.\n    //\n    // The reasons for ensuring all instances have the same hidden class are\n    // further discussed in this blog post from Benedikt Meurer:\n    // https://benediktmeurer.de/2018/03/23/impact-of-polymorphism-on-component-based-frameworks-like-react/\n    super(destination);\n    this._next = onNext\n      ? function (this: OperatorSubscriber<T>, value: T) {\n          try {\n            onNext(value);\n          } catch (err) {\n            destination.error(err);\n          }\n        }\n      : super._next;\n    this._error = onError\n      ? function (this: OperatorSubscriber<T>, err: any) {\n          try {\n            onError(err);\n          } catch (err) {\n            // Send any errors that occur down stream.\n            destination.error(err);\n          } finally {\n            // Ensure finalization.\n            this.unsubscribe();\n          }\n        }\n      : super._error;\n    this._complete = onComplete\n      ? function (this: OperatorSubscriber<T>) {\n          try {\n            onComplete();\n          } catch (err) {\n            // Send any errors that occur down stream.\n            destination.error(err);\n          } finally {\n            // Ensure finalization.\n            this.unsubscribe();\n          }\n        }\n      : super._complete;\n  }\n\n  unsubscribe() {\n    if (!this.shouldUnsubscribe || this.shouldUnsubscribe()) {\n      const { closed } = this;\n      super.unsubscribe();\n      // Execute additional teardown if we have any and we didn't already do so.\n      !closed && this.onFinalize?.();\n    }\n  }\n}\n", "import { Subscription } from '../Subscription';\n\ninterface AnimationFrameProvider {\n  schedule(callback: FrameRequestCallback): Subscription;\n  requestAnimationFrame: typeof requestAnimationFrame;\n  cancelAnimationFrame: typeof cancelAnimationFrame;\n  delegate:\n    | {\n        requestAnimationFrame: typeof requestAnimationFrame;\n        cancelAnimationFrame: typeof cancelAnimationFrame;\n      }\n    | undefined;\n}\n\nexport const animationFrameProvider: AnimationFrameProvider = {\n  // When accessing the delegate, use the variable rather than `this` so that\n  // the functions can be called without being bound to the provider.\n  schedule(callback) {\n    let request = requestAnimationFrame;\n    let cancel: typeof cancelAnimationFrame | undefined = cancelAnimationFrame;\n    const { delegate } = animationFrameProvider;\n    if (delegate) {\n      request = delegate.requestAnimationFrame;\n      cancel = delegate.cancelAnimationFrame;\n    }\n    const handle = request((timestamp) => {\n      // Clear the cancel function. The request has been fulfilled, so\n      // attempting to cancel the request upon unsubscription would be\n      // pointless.\n      cancel = undefined;\n      callback(timestamp);\n    });\n    return new Subscription(() => cancel?.(handle));\n  },\n  requestAnimationFrame(...args) {\n    const { delegate } = animationFrameProvider;\n    return (delegate?.requestAnimationFrame || requestAnimationFrame)(...args);\n  },\n  cancelAnimationFrame(...args) {\n    const { delegate } = animationFrameProvider;\n    return (delegate?.cancelAnimationFrame || cancelAnimationFrame)(...args);\n  },\n  delegate: undefined,\n};\n", "import { createErrorClass } from './createErrorClass';\n\nexport interface ObjectUnsubscribedError extends Error {}\n\nexport interface ObjectUnsubscribedErrorCtor {\n  /**\n   * @deprecated Internal implementation detail. Do not construct error instances.\n   * Cannot be tagged as internal: https://github.com/ReactiveX/rxjs/issues/6269\n   */\n  new (): ObjectUnsubscribedError;\n}\n\n/**\n * An error thrown when an action is invalid because the object has been\n * unsubscribed.\n *\n * @see {@link Subject}\n * @see {@link BehaviorSubject}\n *\n * @class ObjectUnsubscribedError\n */\nexport const ObjectUnsubscribedError: ObjectUnsubscribedErrorCtor = createErrorClass(\n  (_super) =>\n    function ObjectUnsubscribedErrorImpl(this: any) {\n      _super(this);\n      this.name = 'ObjectUnsubscribedError';\n      this.message = 'object unsubscribed';\n    }\n);\n", "import { Operator } from './Operator';\nimport { Observable } from './Observable';\nimport { Subscriber } from './Subscriber';\nimport { Subscription, EMPTY_SUBSCRIPTION } from './Subscription';\nimport { Observer, SubscriptionLike, TeardownLogic } from './types';\nimport { ObjectUnsubscribedError } from './util/ObjectUnsubscribedError';\nimport { arrRemove } from './util/arrRemove';\nimport { errorContext } from './util/errorContext';\n\n/**\n * A Subject is a special type of Observable that allows values to be\n * multicasted to many Observers. Subjects are like EventEmitters.\n *\n * Every Subject is an Observable and an Observer. You can subscribe to a\n * Subject, and you can call next to feed values as well as error and complete.\n */\nexport class Subject<T> extends Observable<T> implements SubscriptionLike {\n  closed = false;\n\n  private currentObservers: Observer<T>[] | null = null;\n\n  /** @deprecated Internal implementation detail, do not use directly. Will be made internal in v8. */\n  observers: Observer<T>[] = [];\n  /** @deprecated Internal implementation detail, do not use directly. Will be made internal in v8. */\n  isStopped = false;\n  /** @deprecated Internal implementation detail, do not use directly. Will be made internal in v8. */\n  hasError = false;\n  /** @deprecated Internal implementation detail, do not use directly. Will be made internal in v8. */\n  thrownError: any = null;\n\n  /**\n   * Creates a \"subject\" by basically gluing an observer to an observable.\n   *\n   * @nocollapse\n   * @deprecated Recommended you do not use. Will be removed at some point in the future. Plans for replacement still under discussion.\n   */\n  static create: (...args: any[]) => any = <T>(destination: Observer<T>, source: Observable<T>): AnonymousSubject<T> => {\n    return new AnonymousSubject<T>(destination, source);\n  };\n\n  constructor() {\n    // NOTE: This must be here to obscure Observable's constructor.\n    super();\n  }\n\n  /** @deprecated Internal implementation detail, do not use directly. Will be made internal in v8. */\n  lift<R>(operator: Operator<T, R>): Observable<R> {\n    const subject = new AnonymousSubject(this, this);\n    subject.operator = operator as any;\n    return subject as any;\n  }\n\n  /** @internal */\n  protected _throwIfClosed() {\n    if (this.closed) {\n      throw new ObjectUnsubscribedError();\n    }\n  }\n\n  next(value: T) {\n    errorContext(() => {\n      this._throwIfClosed();\n      if (!this.isStopped) {\n        if (!this.currentObservers) {\n          this.currentObservers = Array.from(this.observers);\n        }\n        for (const observer of this.currentObservers) {\n          observer.next(value);\n        }\n      }\n    });\n  }\n\n  error(err: any) {\n    errorContext(() => {\n      this._throwIfClosed();\n      if (!this.isStopped) {\n        this.hasError = this.isStopped = true;\n        this.thrownError = err;\n        const { observers } = this;\n        while (observers.length) {\n          observers.shift()!.error(err);\n        }\n      }\n    });\n  }\n\n  complete() {\n    errorContext(() => {\n      this._throwIfClosed();\n      if (!this.isStopped) {\n        this.isStopped = true;\n        const { observers } = this;\n        while (observers.length) {\n          observers.shift()!.complete();\n        }\n      }\n    });\n  }\n\n  unsubscribe() {\n    this.isStopped = this.closed = true;\n    this.observers = this.currentObservers = null!;\n  }\n\n  get observed() {\n    return this.observers?.length > 0;\n  }\n\n  /** @internal */\n  protected _trySubscribe(subscriber: Subscriber<T>): TeardownLogic {\n    this._throwIfClosed();\n    return super._trySubscribe(subscriber);\n  }\n\n  /** @internal */\n  protected _subscribe(subscriber: Subscriber<T>): Subscription {\n    this._throwIfClosed();\n    this._checkFinalizedStatuses(subscriber);\n    return this._innerSubscribe(subscriber);\n  }\n\n  /** @internal */\n  protected _innerSubscribe(subscriber: Subscriber<any>) {\n    const { hasError, isStopped, observers } = this;\n    if (hasError || isStopped) {\n      return EMPTY_SUBSCRIPTION;\n    }\n    this.currentObservers = null;\n    observers.push(subscriber);\n    return new Subscription(() => {\n      this.currentObservers = null;\n      arrRemove(observers, subscriber);\n    });\n  }\n\n  /** @internal */\n  protected _checkFinalizedStatuses(subscriber: Subscriber<any>) {\n    const { hasError, thrownError, isStopped } = this;\n    if (hasError) {\n      subscriber.error(thrownError);\n    } else if (isStopped) {\n      subscriber.complete();\n    }\n  }\n\n  /**\n   * Creates a new Observable with this Subject as the source. You can do this\n   * to create custom Observer-side logic of the Subject and conceal it from\n   * code that uses the Observable.\n   * @return {Observable} Observable that the Subject casts to\n   */\n  asObservable(): Observable<T> {\n    const observable: any = new Observable<T>();\n    observable.source = this;\n    return observable;\n  }\n}\n\n/**\n * @class AnonymousSubject<T>\n */\nexport class AnonymousSubject<T> extends Subject<T> {\n  constructor(\n    /** @deprecated Internal implementation detail, do not use directly. Will be made internal in v8. */\n    public destination?: Observer<T>,\n    source?: Observable<T>\n  ) {\n    super();\n    this.source = source;\n  }\n\n  next(value: T) {\n    this.destination?.next?.(value);\n  }\n\n  error(err: any) {\n    this.destination?.error?.(err);\n  }\n\n  complete() {\n    this.destination?.complete?.();\n  }\n\n  /** @internal */\n  protected _subscribe(subscriber: Subscriber<T>): Subscription {\n    return this.source?.subscribe(subscriber) ?? EMPTY_SUBSCRIPTION;\n  }\n}\n", "import { TimestampProvider } from '../types';\n\ninterface DateTimestampProvider extends TimestampProvider {\n  delegate: TimestampProvider | undefined;\n}\n\nexport const dateTimestampProvider: DateTimestampProvider = {\n  now() {\n    // Use the variable rather than `this` so that the function can be called\n    // without being bound to the provider.\n    return (dateTimestampProvider.delegate || Date).now();\n  },\n  delegate: undefined,\n};\n", "import { Subject } from './Subject';\nimport { TimestampProvider } from './types';\nimport { Subscriber } from './Subscriber';\nimport { Subscription } from './Subscription';\nimport { dateTimestampProvider } from './scheduler/dateTimestampProvider';\n\n/**\n * A variant of {@link Subject} that \"replays\" old values to new subscribers by emitting them when they first subscribe.\n *\n * `ReplaySubject` has an internal buffer that will store a specified number of values that it has observed. Like `Subject`,\n * `ReplaySubject` \"observes\" values by having them passed to its `next` method. When it observes a value, it will store that\n * value for a time determined by the configuration of the `ReplaySubject`, as passed to its constructor.\n *\n * When a new subscriber subscribes to the `ReplaySubject` instance, it will synchronously emit all values in its buffer in\n * a First-In-First-Out (FIFO) manner. The `ReplaySubject` will also complete, if it has observed completion; and it will\n * error if it has observed an error.\n *\n * There are two main configuration items to be concerned with:\n *\n * 1. `bufferSize` - This will determine how many items are stored in the buffer, defaults to infinite.\n * 2. `windowTime` - The amount of time to hold a value in the buffer before removing it from the buffer.\n *\n * Both configurations may exist simultaneously. So if you would like to buffer a maximum of 3 values, as long as the values\n * are less than 2 seconds old, you could do so with a `new ReplaySubject(3, 2000)`.\n *\n * ### Differences with BehaviorSubject\n *\n * `BehaviorSubject` is similar to `new ReplaySubject(1)`, with a couple of exceptions:\n *\n * 1. `BehaviorSubject` comes \"primed\" with a single value upon construction.\n * 2. `ReplaySubject` will replay values, even after observing an error, where `BehaviorSubject` will not.\n *\n * @see {@link Subject}\n * @see {@link BehaviorSubject}\n * @see {@link shareReplay}\n */\nexport class ReplaySubject<T> extends Subject<T> {\n  private _buffer: (T | number)[] = [];\n  private _infiniteTimeWindow = true;\n\n  /**\n   * @param bufferSize The size of the buffer to replay on subscription\n   * @param windowTime The amount of time the buffered items will stay buffered\n   * @param timestampProvider An object with a `now()` method that provides the current timestamp. This is used to\n   * calculate the amount of time something has been buffered.\n   */\n  constructor(\n    private _bufferSize = Infinity,\n    private _windowTime = Infinity,\n    private _timestampProvider: TimestampProvider = dateTimestampProvider\n  ) {\n    super();\n    this._infiniteTimeWindow = _windowTime === Infinity;\n    this._bufferSize = Math.max(1, _bufferSize);\n    this._windowTime = Math.max(1, _windowTime);\n  }\n\n  next(value: T): void {\n    const { isStopped, _buffer, _infiniteTimeWindow, _timestampProvider, _windowTime } = this;\n    if (!isStopped) {\n      _buffer.push(value);\n      !_infiniteTimeWindow && _buffer.push(_timestampProvider.now() + _windowTime);\n    }\n    this._trimBuffer();\n    super.next(value);\n  }\n\n  /** @internal */\n  protected _subscribe(subscriber: Subscriber<T>): Subscription {\n    this._throwIfClosed();\n    this._trimBuffer();\n\n    const subscription = this._innerSubscribe(subscriber);\n\n    const { _infiniteTimeWindow, _buffer } = this;\n    // We use a copy here, so reentrant code does not mutate our array while we're\n    // emitting it to a new subscriber.\n    const copy = _buffer.slice();\n    for (let i = 0; i < copy.length && !subscriber.closed; i += _infiniteTimeWindow ? 1 : 2) {\n      subscriber.next(copy[i] as T);\n    }\n\n    this._checkFinalizedStatuses(subscriber);\n\n    return subscription;\n  }\n\n  private _trimBuffer() {\n    const { _bufferSize, _timestampProvider, _buffer, _infiniteTimeWindow } = this;\n    // If we don't have an infinite buffer size, and we're over the length,\n    // use splice to truncate the old buffer values off. Note that we have to\n    // double the size for instances where we're not using an infinite time window\n    // because we're storing the values and the timestamps in the same array.\n    const adjustedBufferSize = (_infiniteTimeWindow ? 1 : 2) * _bufferSize;\n    _bufferSize < Infinity && adjustedBufferSize < _buffer.length && _buffer.splice(0, _buffer.length - adjustedBufferSize);\n\n    // Now, if we're not in an infinite time window, remove all values where the time is\n    // older than what is allowed.\n    if (!_infiniteTimeWindow) {\n      const now = _timestampProvider.now();\n      let last = 0;\n      // Search the array for the first timestamp that isn't expired and\n      // truncate the buffer up to that point.\n      for (let i = 1; i < _buffer.length && (_buffer[i] as number) <= now; i += 2) {\n        last = i;\n      }\n      last && _buffer.splice(0, last + 1);\n    }\n  }\n}\n", "import { Scheduler } from '../Scheduler';\nimport { Subscription } from '../Subscription';\nimport { SchedulerAction } from '../types';\n\n/**\n * A unit of work to be executed in a `scheduler`. An action is typically\n * created from within a {@link SchedulerLike} and an RxJS user does not need to concern\n * themselves about creating and manipulating an Action.\n *\n * ```ts\n * class Action<T> extends Subscription {\n *   new (scheduler: Scheduler, work: (state?: T) => void);\n *   schedule(state?: T, delay: number = 0): Subscription;\n * }\n * ```\n *\n * @class Action<T>\n */\nexport class Action<T> extends Subscription {\n  constructor(scheduler: Scheduler, work: (this: SchedulerAction<T>, state?: T) => void) {\n    super();\n  }\n  /**\n   * Schedules this action on its parent {@link SchedulerLike} for execution. May be passed\n   * some context object, `state`. May happen at some point in the future,\n   * according to the `delay` parameter, if specified.\n   * @param {T} [state] Some contextual data that the `work` function uses when\n   * called by the Scheduler.\n   * @param {number} [delay] Time to wait before executing the work, where the\n   * time unit is implicit and defined by the Scheduler.\n   * @return {void}\n   */\n  public schedule(state?: T, delay: number = 0): Subscription {\n    return this;\n  }\n}\n", "import type { TimerHandle } from './timerHandle';\ntype SetIntervalFunction = (handler: () => void, timeout?: number, ...args: any[]) => TimerHandle;\ntype ClearIntervalFunction = (handle: TimerHandle) => void;\n\ninterface IntervalProvider {\n  setInterval: SetIntervalFunction;\n  clearInterval: ClearIntervalFunction;\n  delegate:\n    | {\n        setInterval: SetIntervalFunction;\n        clearInterval: ClearIntervalFunction;\n      }\n    | undefined;\n}\n\nexport const intervalProvider: IntervalProvider = {\n  // When accessing the delegate, use the variable rather than `this` so that\n  // the functions can be called without being bound to the provider.\n  setInterval(handler: () => void, timeout?: number, ...args) {\n    const { delegate } = intervalProvider;\n    if (delegate?.setInterval) {\n      return delegate.setInterval(handler, timeout, ...args);\n    }\n    return setInterval(handler, timeout, ...args);\n  },\n  clearInterval(handle) {\n    const { delegate } = intervalProvider;\n    return (delegate?.clearInterval || clearInterval)(handle as any);\n  },\n  delegate: undefined,\n};\n", "import { Action } from './Action';\nimport { SchedulerAction } from '../types';\nimport { Subscription } from '../Subscription';\nimport { AsyncScheduler } from './AsyncScheduler';\nimport { intervalProvider } from './intervalProvider';\nimport { arrRemove } from '../util/arrRemove';\nimport { TimerHandle } from './timerHandle';\n\nexport class AsyncAction<T> extends Action<T> {\n  public id: TimerHandle | undefined;\n  public state?: T;\n  // @ts-ignore: Property has no initializer and is not definitely assigned\n  public delay: number;\n  protected pending: boolean = false;\n\n  constructor(protected scheduler: AsyncScheduler, protected work: (this: SchedulerAction<T>, state?: T) => void) {\n    super(scheduler, work);\n  }\n\n  public schedule(state?: T, delay: number = 0): Subscription {\n    if (this.closed) {\n      return this;\n    }\n\n    // Always replace the current state with the new state.\n    this.state = state;\n\n    const id = this.id;\n    const scheduler = this.scheduler;\n\n    //\n    // Important implementation note:\n    //\n    // Actions only execute once by default, unless rescheduled from within the\n    // scheduled callback. This allows us to implement single and repeat\n    // actions via the same code path, without adding API surface area, as well\n    // as mimic traditional recursion but across asynchronous boundaries.\n    //\n    // However, JS runtimes and timers distinguish between intervals achieved by\n    // serial `setTimeout` calls vs. a single `setInterval` call. An interval of\n    // serial `setTimeout` calls can be individually delayed, which delays\n    // scheduling the next `setTimeout`, and so on. `setInterval` attempts to\n    // guarantee the interval callback will be invoked more precisely to the\n    // interval period, regardless of load.\n    //\n    // Therefore, we use `setInterval` to schedule single and repeat actions.\n    // If the action reschedules itself with the same delay, the interval is not\n    // canceled. If the action doesn't reschedule, or reschedules with a\n    // different delay, the interval will be canceled after scheduled callback\n    // execution.\n    //\n    if (id != null) {\n      this.id = this.recycleAsyncId(scheduler, id, delay);\n    }\n\n    // Set the pending flag indicating that this action has been scheduled, or\n    // has recursively rescheduled itself.\n    this.pending = true;\n\n    this.delay = delay;\n    // If this action has already an async Id, don't request a new one.\n    this.id = this.id ?? this.requestAsyncId(scheduler, this.id, delay);\n\n    return this;\n  }\n\n  protected requestAsyncId(scheduler: AsyncScheduler, _id?: TimerHandle, delay: number = 0): TimerHandle {\n    return intervalProvider.setInterval(scheduler.flush.bind(scheduler, this), delay);\n  }\n\n  protected recycleAsyncId(_scheduler: AsyncScheduler, id?: TimerHandle, delay: number | null = 0): TimerHandle | undefined {\n    // If this action is rescheduled with the same delay time, don't clear the interval id.\n    if (delay != null && this.delay === delay && this.pending === false) {\n      return id;\n    }\n    // Otherwise, if the action's delay time is different from the current delay,\n    // or the action has been rescheduled before it's executed, clear the interval id\n    if (id != null) {\n      intervalProvider.clearInterval(id);\n    }\n\n    return undefined;\n  }\n\n  /**\n   * Immediately executes this action and the `work` it contains.\n   * @return {any}\n   */\n  public execute(state: T, delay: number): any {\n    if (this.closed) {\n      return new Error('executing a cancelled action');\n    }\n\n    this.pending = false;\n    const error = this._execute(state, delay);\n    if (error) {\n      return error;\n    } else if (this.pending === false && this.id != null) {\n      // Dequeue if the action didn't reschedule itself. Don't call\n      // unsubscribe(), because the action could reschedule later.\n      // For example:\n      // ```\n      // scheduler.schedule(function doWork(counter) {\n      //   /* ... I'm a busy worker bee ... */\n      //   var originalAction = this;\n      //   /* wait 100ms before rescheduling the action */\n      //   setTimeout(function () {\n      //     originalAction.schedule(counter + 1);\n      //   }, 100);\n      // }, 1000);\n      // ```\n      this.id = this.recycleAsyncId(this.scheduler, this.id, null);\n    }\n  }\n\n  protected _execute(state: T, _delay: number): any {\n    let errored: boolean = false;\n    let errorValue: any;\n    try {\n      this.work(state);\n    } catch (e) {\n      errored = true;\n      // HACK: Since code elsewhere is relying on the \"truthiness\" of the\n      // return here, we can't have it return \"\" or 0 or false.\n      // TODO: Clean this up when we refactor schedulers mid-version-8 or so.\n      errorValue = e ? e : new Error('Scheduled action threw falsy error');\n    }\n    if (errored) {\n      this.unsubscribe();\n      return errorValue;\n    }\n  }\n\n  unsubscribe() {\n    if (!this.closed) {\n      const { id, scheduler } = this;\n      const { actions } = scheduler;\n\n      this.work = this.state = this.scheduler = null!;\n      this.pending = false;\n\n      arrRemove(actions, this);\n      if (id != null) {\n        this.id = this.recycleAsyncId(scheduler, id, null);\n      }\n\n      this.delay = null!;\n      super.unsubscribe();\n    }\n  }\n}\n", "import { Action } from './scheduler/Action';\nimport { Subscription } from './Subscription';\nimport { SchedulerLike, SchedulerAction } from './types';\nimport { dateTimestampProvider } from './scheduler/dateTimestampProvider';\n\n/**\n * An execution context and a data structure to order tasks and schedule their\n * execution. Provides a notion of (potentially virtual) time, through the\n * `now()` getter method.\n *\n * Each unit of work in a Scheduler is called an `Action`.\n *\n * ```ts\n * class Scheduler {\n *   now(): number;\n *   schedule(work, delay?, state?): Subscription;\n * }\n * ```\n *\n * @class Scheduler\n * @deprecated Scheduler is an internal implementation detail of RxJS, and\n * should not be used directly. Rather, create your own class and implement\n * {@link SchedulerLike}. Will be made internal in v8.\n */\nexport class Scheduler implements SchedulerLike {\n  public static now: () => number = dateTimestampProvider.now;\n\n  constructor(private schedulerActionCtor: typeof Action, now: () => number = Scheduler.now) {\n    this.now = now;\n  }\n\n  /**\n   * A getter method that returns a number representing the current time\n   * (at the time this function was called) according to the scheduler's own\n   * internal clock.\n   * @return {number} A number that represents the current time. May or may not\n   * have a relation to wall-clock time. May or may not refer to a time unit\n   * (e.g. milliseconds).\n   */\n  public now: () => number;\n\n  /**\n   * Schedules a function, `work`, for execution. May happen at some point in\n   * the future, according to the `delay` parameter, if specified. May be passed\n   * some context object, `state`, which will be passed to the `work` function.\n   *\n   * The given arguments will be processed an stored as an Action object in a\n   * queue of actions.\n   *\n   * @param {function(state: ?T): ?Subscription} work A function representing a\n   * task, or some unit of work to be executed by the Scheduler.\n   * @param {number} [delay] Time to wait before executing the work, where the\n   * time unit is implicit and defined by the Scheduler itself.\n   * @param {T} [state] Some contextual data that the `work` function uses when\n   * called by the Scheduler.\n   * @return {Subscription} A subscription in order to be able to unsubscribe\n   * the scheduled work.\n   */\n  public schedule<T>(work: (this: SchedulerAction<T>, state?: T) => void, delay: number = 0, state?: T): Subscription {\n    return new this.schedulerActionCtor<T>(this, work).schedule(state, delay);\n  }\n}\n", "import { Scheduler } from '../Scheduler';\nimport { Action } from './Action';\nimport { AsyncAction } from './AsyncAction';\nimport { TimerHandle } from './timerHandle';\n\nexport class AsyncScheduler extends Scheduler {\n  public actions: Array<AsyncAction<any>> = [];\n  /**\n   * A flag to indicate whether the Scheduler is currently executing a batch of\n   * queued actions.\n   * @type {boolean}\n   * @internal\n   */\n  public _active: boolean = false;\n  /**\n   * An internal ID used to track the latest asynchronous task such as those\n   * coming from `setTimeout`, `setInterval`, `requestAnimationFrame`, and\n   * others.\n   * @type {any}\n   * @internal\n   */\n  public _scheduled: TimerHandle | undefined;\n\n  constructor(SchedulerAction: typeof Action, now: () => number = Scheduler.now) {\n    super(SchedulerAction, now);\n  }\n\n  public flush(action: AsyncAction<any>): void {\n    const { actions } = this;\n\n    if (this._active) {\n      actions.push(action);\n      return;\n    }\n\n    let error: any;\n    this._active = true;\n\n    do {\n      if ((error = action.execute(action.state, action.delay))) {\n        break;\n      }\n    } while ((action = actions.shift()!)); // exhaust the scheduler queue\n\n    this._active = false;\n\n    if (error) {\n      while ((action = actions.shift()!)) {\n        action.unsubscribe();\n      }\n      throw error;\n    }\n  }\n}\n", "import { AsyncAction } from './AsyncAction';\nimport { AsyncScheduler } from './AsyncScheduler';\n\n/**\n *\n * Async Scheduler\n *\n * <span class=\"informal\">Schedule task as if you used setTimeout(task, duration)</span>\n *\n * `async` scheduler schedules tasks asynchronously, by putting them on the JavaScript\n * event loop queue. It is best used to delay tasks in time or to schedule tasks repeating\n * in intervals.\n *\n * If you just want to \"defer\" task, that is to perform it right after currently\n * executing synchronous code ends (commonly achieved by `setTimeout(deferredTask, 0)`),\n * better choice will be the {@link asapScheduler} scheduler.\n *\n * ## Examples\n * Use async scheduler to delay task\n * ```ts\n * import { asyncScheduler } from 'rxjs';\n *\n * const task = () => console.log('it works!');\n *\n * asyncScheduler.schedule(task, 2000);\n *\n * // After 2 seconds logs:\n * // \"it works!\"\n * ```\n *\n * Use async scheduler to repeat task in intervals\n * ```ts\n * import { asyncScheduler } from 'rxjs';\n *\n * function task(state) {\n *   console.log(state);\n *   this.schedule(state + 1, 1000); // `this` references currently executing Action,\n *                                   // which we reschedule with new state and delay\n * }\n *\n * asyncScheduler.schedule(task, 3000, 0);\n *\n * // Logs:\n * // 0 after 3s\n * // 1 after 4s\n * // 2 after 5s\n * // 3 after 6s\n * ```\n */\n\nexport const asyncScheduler = new AsyncScheduler(AsyncAction);\n\n/**\n * @deprecated Renamed to {@link asyncScheduler}. Will be removed in v8.\n */\nexport const async = asyncScheduler;\n", "import { AsyncAction } from './AsyncAction';\nimport { AnimationFrameScheduler } from './AnimationFrameScheduler';\nimport { SchedulerAction } from '../types';\nimport { animationFrameProvider } from './animationFrameProvider';\nimport { TimerHandle } from './timerHandle';\n\nexport class AnimationFrameAction<T> extends AsyncAction<T> {\n  constructor(protected scheduler: AnimationFrameScheduler, protected work: (this: SchedulerAction<T>, state?: T) => void) {\n    super(scheduler, work);\n  }\n\n  protected requestAsyncId(scheduler: AnimationFrameScheduler, id?: TimerHandle, delay: number = 0): TimerHandle {\n    // If delay is greater than 0, request as an async action.\n    if (delay !== null && delay > 0) {\n      return super.requestAsyncId(scheduler, id, delay);\n    }\n    // Push the action to the end of the scheduler queue.\n    scheduler.actions.push(this);\n    // If an animation frame has already been requested, don't request another\n    // one. If an animation frame hasn't been requested yet, request one. Return\n    // the current animation frame request id.\n    return scheduler._scheduled || (scheduler._scheduled = animationFrameProvider.requestAnimationFrame(() => scheduler.flush(undefined)));\n  }\n\n  protected recycleAsyncId(scheduler: AnimationFrameScheduler, id?: TimerHandle, delay: number = 0): TimerHandle | undefined {\n    // If delay exists and is greater than 0, or if the delay is null (the\n    // action wasn't rescheduled) but was originally scheduled as an async\n    // action, then recycle as an async action.\n    if (delay != null ? delay > 0 : this.delay > 0) {\n      return super.recycleAsyncId(scheduler, id, delay);\n    }\n    // If the scheduler queue has no remaining actions with the same async id,\n    // cancel the requested animation frame and set the scheduled flag to\n    // undefined so the next AnimationFrameAction will request its own.\n    const { actions } = scheduler;\n    if (id != null && actions[actions.length - 1]?.id !== id) {\n      animationFrameProvider.cancelAnimationFrame(id as number);\n      scheduler._scheduled = undefined;\n    }\n    // Return undefined so the action knows to request a new async id if it's rescheduled.\n    return undefined;\n  }\n}\n", "import { AsyncAction } from './AsyncAction';\nimport { AsyncScheduler } from './AsyncScheduler';\n\nexport class AnimationFrameScheduler extends AsyncScheduler {\n  public flush(action?: AsyncAction<any>): void {\n    this._active = true;\n    // The async id that effects a call to flush is stored in _scheduled.\n    // Before executing an action, it's necessary to check the action's async\n    // id to determine whether it's supposed to be executed in the current\n    // flush.\n    // Previous implementations of this method used a count to determine this,\n    // but that was unsound, as actions that are unsubscribed - i.e. cancelled -\n    // are removed from the actions array and that can shift actions that are\n    // scheduled to be executed in a subsequent flush into positions at which\n    // they are executed within the current flush.\n    const flushId = this._scheduled;\n    this._scheduled = undefined;\n\n    const { actions } = this;\n    let error: any;\n    action = action || actions.shift()!;\n\n    do {\n      if ((error = action.execute(action.state, action.delay))) {\n        break;\n      }\n    } while ((action = actions[0]) && action.id === flushId && actions.shift());\n\n    this._active = false;\n\n    if (error) {\n      while ((action = actions[0]) && action.id === flushId && actions.shift()) {\n        action.unsubscribe();\n      }\n      throw error;\n    }\n  }\n}\n", "import { AnimationFrameAction } from './AnimationFrameAction';\nimport { AnimationFrameScheduler } from './AnimationFrameScheduler';\n\n/**\n *\n * Animation Frame Scheduler\n *\n * <span class=\"informal\">Perform task when `window.requestAnimationFrame` would fire</span>\n *\n * When `animationFrame` scheduler is used with delay, it will fall back to {@link asyncScheduler} scheduler\n * behaviour.\n *\n * Without delay, `animationFrame` scheduler can be used to create smooth browser animations.\n * It makes sure scheduled task will happen just before next browser content repaint,\n * thus performing animations as efficiently as possible.\n *\n * ## Example\n * Schedule div height animation\n * ```ts\n * // html: <div style=\"background: #0ff;\"></div>\n * import { animationFrameScheduler } from 'rxjs';\n *\n * const div = document.querySelector('div');\n *\n * animationFrameScheduler.schedule(function(height) {\n *   div.style.height = height + \"px\";\n *\n *   this.schedule(height + 1);  // `this` references currently executing Action,\n *                               // which we reschedule with new state\n * }, 0, 0);\n *\n * // You will see a div element growing in height\n * ```\n */\n\nexport const animationFrameScheduler = new AnimationFrameScheduler(AnimationFrameAction);\n\n/**\n * @deprecated Renamed to {@link animationFrameScheduler}. Will be removed in v8.\n */\nexport const animationFrame = animationFrameScheduler;\n", "import { Observable } from '../Observable';\nimport { SchedulerLike } from '../types';\n\n/**\n * A simple Observable that emits no items to the Observer and immediately\n * emits a complete notification.\n *\n * <span class=\"informal\">Just emits 'complete', and nothing else.</span>\n *\n * ![](empty.png)\n *\n * A simple Observable that only emits the complete notification. It can be used\n * for composing with other Observables, such as in a {@link mergeMap}.\n *\n * ## Examples\n *\n * Log complete notification\n *\n * ```ts\n * import { EMPTY } from 'rxjs';\n *\n * EMPTY.subscribe({\n *   next: () => console.log('Next'),\n *   complete: () => console.log('Complete!')\n * });\n *\n * // Outputs\n * // Complete!\n * ```\n *\n * Emit the number 7, then complete\n *\n * ```ts\n * import { EMPTY, startWith } from 'rxjs';\n *\n * const result = EMPTY.pipe(startWith(7));\n * result.subscribe(x => console.log(x));\n *\n * // Outputs\n * // 7\n * ```\n *\n * Map and flatten only odd numbers to the sequence `'a'`, `'b'`, `'c'`\n *\n * ```ts\n * import { interval, mergeMap, of, EMPTY } from 'rxjs';\n *\n * const interval$ = interval(1000);\n * const result = interval$.pipe(\n *   mergeMap(x => x % 2 === 1 ? of('a', 'b', 'c') : EMPTY),\n * );\n * result.subscribe(x => console.log(x));\n *\n * // Results in the following to the console:\n * // x is equal to the count on the interval, e.g. (0, 1, 2, 3, ...)\n * // x will occur every 1000ms\n * // if x % 2 is equal to 1, print a, b, c (each on its own)\n * // if x % 2 is not equal to 1, nothing will be output\n * ```\n *\n * @see {@link Observable}\n * @see {@link NEVER}\n * @see {@link of}\n * @see {@link throwError}\n */\nexport const EMPTY = new Observable<never>((subscriber) => subscriber.complete());\n\n/**\n * @param scheduler A {@link SchedulerLike} to use for scheduling\n * the emission of the complete notification.\n * @deprecated Replaced with the {@link EMPTY} constant or {@link scheduled} (e.g. `scheduled([], scheduler)`). Will be removed in v8.\n */\nexport function empty(scheduler?: SchedulerLike) {\n  return scheduler ? emptyScheduled(scheduler) : EMPTY;\n}\n\nfunction emptyScheduled(scheduler: SchedulerLike) {\n  return new Observable<never>((subscriber) => scheduler.schedule(() => subscriber.complete()));\n}\n", "import { SchedulerLike } from '../types';\nimport { isFunction } from './isFunction';\n\nexport function isScheduler(value: any): value is SchedulerLike {\n  return value && isFunction(value.schedule);\n}\n", "import { SchedulerLike } from '../types';\nimport { isFunction } from './isFunction';\nimport { isScheduler } from './isScheduler';\n\nfunction last<T>(arr: T[]): T | undefined {\n  return arr[arr.length - 1];\n}\n\nexport function popResultSelector(args: any[]): ((...args: unknown[]) => unknown) | undefined {\n  return isFunction(last(args)) ? args.pop() : undefined;\n}\n\nexport function popScheduler(args: any[]): SchedulerLike | undefined {\n  return isScheduler(last(args)) ? args.pop() : undefined;\n}\n\nexport function popNumber(args: any[], defaultValue: number): number {\n  return typeof last(args) === 'number' ? args.pop()! : defaultValue;\n}\n", "export const isArrayLike = (<T>(x: any): x is ArrayLike<T> => x && typeof x.length === 'number' && typeof x !== 'function');", "import { isFunction } from \"./isFunction\";\n\n/**\n * Tests to see if the object is \"thennable\".\n * @param value the object to test\n */\nexport function isPromise(value: any): value is PromiseLike<any> {\n  return isFunction(value?.then);\n}\n", "import { InteropObservable } from '../types';\nimport { observable as Symbol_observable } from '../symbol/observable';\nimport { isFunction } from './isFunction';\n\n/** Identifies an input as being Observable (but not necessary an Rx Observable) */\nexport function isInteropObservable(input: any): input is InteropObservable<any> {\n  return isFunction(input[Symbol_observable]);\n}\n", "import { isFunction } from './isFunction';\n\nexport function isAsyncIterable<T>(obj: any): obj is AsyncIterable<T> {\n  return Symbol.asyncIterator && isFunction(obj?.[Symbol.asyncIterator]);\n}\n", "/**\n * Creates the TypeError to throw if an invalid object is passed to `from` or `scheduled`.\n * @param input The object that was passed.\n */\nexport function createInvalidObservableTypeError(input: any) {\n  // TODO: We should create error codes that can be looked up, so this can be less verbose.\n  return new TypeError(\n    `You provided ${\n      input !== null && typeof input === 'object' ? 'an invalid object' : `'${input}'`\n    } where a stream was expected. You can provide an Observable, Promise, ReadableStream, Array, AsyncIterable, or Iterable.`\n  );\n}\n", "export function getSymbolIterator(): symbol {\n  if (typeof Symbol !== 'function' || !Symbol.iterator) {\n    return '@@iterator' as any;\n  }\n\n  return Symbol.iterator;\n}\n\nexport const iterator = getSymbolIterator();\n", "import { iterator as Symbol_iterator } from '../symbol/iterator';\nimport { isFunction } from './isFunction';\n\n/** Identifies an input as being an Iterable */\nexport function isIterable(input: any): input is Iterable<any> {\n  return isFunction(input?.[Symbol_iterator]);\n}\n", "import { ReadableStreamLike } from '../types';\nimport { isFunction } from './isFunction';\n\nexport async function* readableStreamLikeToAsyncGenerator<T>(readableStream: ReadableStreamLike<T>): AsyncGenerator<T> {\n  const reader = readableStream.getReader();\n  try {\n    while (true) {\n      const { value, done } = await reader.read();\n      if (done) {\n        return;\n      }\n      yield value!;\n    }\n  } finally {\n    reader.releaseLock();\n  }\n}\n\nexport function isReadableStreamLike<T>(obj: any): obj is ReadableStreamLike<T> {\n  // We don't want to use instanceof checks because they would return\n  // false for instances from another Realm, like an <iframe>.\n  return isFunction(obj?.getReader);\n}\n", "import { isArrayLike } from '../util/isArrayLike';\nimport { isPromise } from '../util/isPromise';\nimport { Observable } from '../Observable';\nimport { ObservableInput, ObservedValueOf, ReadableStreamLike } from '../types';\nimport { isInteropObservable } from '../util/isInteropObservable';\nimport { isAsyncIterable } from '../util/isAsyncIterable';\nimport { createInvalidObservableTypeError } from '../util/throwUnobservableError';\nimport { isIterable } from '../util/isIterable';\nimport { isReadableStreamLike, readableStreamLikeToAsyncGenerator } from '../util/isReadableStreamLike';\nimport { Subscriber } from '../Subscriber';\nimport { isFunction } from '../util/isFunction';\nimport { reportUnhandledError } from '../util/reportUnhandledError';\nimport { observable as Symbol_observable } from '../symbol/observable';\n\nexport function innerFrom<O extends ObservableInput<any>>(input: O): Observable<ObservedValueOf<O>>;\nexport function innerFrom<T>(input: ObservableInput<T>): Observable<T> {\n  if (input instanceof Observable) {\n    return input;\n  }\n  if (input != null) {\n    if (isInteropObservable(input)) {\n      return fromInteropObservable(input);\n    }\n    if (isArrayLike(input)) {\n      return fromArrayLike(input);\n    }\n    if (isPromise(input)) {\n      return fromPromise(input);\n    }\n    if (isAsyncIterable(input)) {\n      return fromAsyncIterable(input);\n    }\n    if (isIterable(input)) {\n      return fromIterable(input);\n    }\n    if (isReadableStreamLike(input)) {\n      return fromReadableStreamLike(input);\n    }\n  }\n\n  throw createInvalidObservableTypeError(input);\n}\n\n/**\n * Creates an RxJS Observable from an object that implements `Symbol.observable`.\n * @param obj An object that properly implements `Symbol.observable`.\n */\nexport function fromInteropObservable<T>(obj: any) {\n  return new Observable((subscriber: Subscriber<T>) => {\n    const obs = obj[Symbol_observable]();\n    if (isFunction(obs.subscribe)) {\n      return obs.subscribe(subscriber);\n    }\n    // Should be caught by observable subscribe function error handling.\n    throw new TypeError('Provided object does not correctly implement Symbol.observable');\n  });\n}\n\n/**\n * Synchronously emits the values of an array like and completes.\n * This is exported because there are creation functions and operators that need to\n * make direct use of the same logic, and there's no reason to make them run through\n * `from` conditionals because we *know* they're dealing with an array.\n * @param array The array to emit values from\n */\nexport function fromArrayLike<T>(array: ArrayLike<T>) {\n  return new Observable((subscriber: Subscriber<T>) => {\n    // Loop over the array and emit each value. Note two things here:\n    // 1. We're making sure that the subscriber is not closed on each loop.\n    //    This is so we don't continue looping over a very large array after\n    //    something like a `take`, `takeWhile`, or other synchronous unsubscription\n    //    has already unsubscribed.\n    // 2. In this form, reentrant code can alter that array we're looping over.\n    //    This is a known issue, but considered an edge case. The alternative would\n    //    be to copy the array before executing the loop, but this has\n    //    performance implications.\n    for (let i = 0; i < array.length && !subscriber.closed; i++) {\n      subscriber.next(array[i]);\n    }\n    subscriber.complete();\n  });\n}\n\nexport function fromPromise<T>(promise: PromiseLike<T>) {\n  return new Observable((subscriber: Subscriber<T>) => {\n    promise\n      .then(\n        (value) => {\n          if (!subscriber.closed) {\n            subscriber.next(value);\n            subscriber.complete();\n          }\n        },\n        (err: any) => subscriber.error(err)\n      )\n      .then(null, reportUnhandledError);\n  });\n}\n\nexport function fromIterable<T>(iterable: Iterable<T>) {\n  return new Observable((subscriber: Subscriber<T>) => {\n    for (const value of iterable) {\n      subscriber.next(value);\n      if (subscriber.closed) {\n        return;\n      }\n    }\n    subscriber.complete();\n  });\n}\n\nexport function fromAsyncIterable<T>(asyncIterable: AsyncIterable<T>) {\n  return new Observable((subscriber: Subscriber<T>) => {\n    process(asyncIterable, subscriber).catch((err) => subscriber.error(err));\n  });\n}\n\nexport function fromReadableStreamLike<T>(readableStream: ReadableStreamLike<T>) {\n  return fromAsyncIterable(readableStreamLikeToAsyncGenerator(readableStream));\n}\n\nasync function process<T>(asyncIterable: AsyncIterable<T>, subscriber: Subscriber<T>) {\n  for await (const value of asyncIterable) {\n    subscriber.next(value);\n    // A side-effect may have closed our subscriber,\n    // check before the next iteration.\n    if (subscriber.closed) {\n      return;\n    }\n  }\n  subscriber.complete();\n}\n", "import { Subscription } from '../Subscription';\nimport { SchedulerAction, SchedulerLike } from '../types';\n\nexport function executeSchedule(\n  parentSubscription: Subscription,\n  scheduler: SchedulerLike,\n  work: () => void,\n  delay: number,\n  repeat: true\n): void;\nexport function executeSchedule(\n  parentSubscription: Subscription,\n  scheduler: SchedulerLike,\n  work: () => void,\n  delay?: number,\n  repeat?: false\n): Subscription;\n\nexport function executeSchedule(\n  parentSubscription: Subscription,\n  scheduler: SchedulerLike,\n  work: () => void,\n  delay = 0,\n  repeat = false\n): Subscription | void {\n  const scheduleSubscription = scheduler.schedule(function (this: SchedulerAction<any>) {\n    work();\n    if (repeat) {\n      parentSubscription.add(this.schedule(null, delay));\n    } else {\n      this.unsubscribe();\n    }\n  }, delay);\n\n  parentSubscription.add(scheduleSubscription);\n\n  if (!repeat) {\n    // Because user-land scheduler implementations are unlikely to properly reuse\n    // Actions for repeat scheduling, we can't trust that the returned subscription\n    // will control repeat subscription scenarios. So we're trying to avoid using them\n    // incorrectly within this library.\n    return scheduleSubscription;\n  }\n}\n", "/** @prettier */\nimport { MonoTypeOperatorFunction, SchedulerLike } from '../types';\nimport { executeSchedule } from '../util/executeSchedule';\nimport { operate } from '../util/lift';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\n\n/**\n * Re-emits all notifications from source Observable with specified scheduler.\n *\n * <span class=\"informal\">Ensure a specific scheduler is used, from outside of an Observable.</span>\n *\n * `observeOn` is an operator that accepts a scheduler as a first parameter, which will be used to reschedule\n * notifications emitted by the source Observable. It might be useful, if you do not have control over\n * internal scheduler of a given Observable, but want to control when its values are emitted nevertheless.\n *\n * Returned Observable emits the same notifications (nexted values, complete and error events) as the source Observable,\n * but rescheduled with provided scheduler. Note that this doesn't mean that source Observables internal\n * scheduler will be replaced in any way. Original scheduler still will be used, but when the source Observable emits\n * notification, it will be immediately scheduled again - this time with scheduler passed to `observeOn`.\n * An anti-pattern would be calling `observeOn` on Observable that emits lots of values synchronously, to split\n * that emissions into asynchronous chunks. For this to happen, scheduler would have to be passed into the source\n * Observable directly (usually into the operator that creates it). `observeOn` simply delays notifications a\n * little bit more, to ensure that they are emitted at expected moments.\n *\n * As a matter of fact, `observeOn` accepts second parameter, which specifies in milliseconds with what delay notifications\n * will be emitted. The main difference between {@link delay} operator and `observeOn` is that `observeOn`\n * will delay all notifications - including error notifications - while `delay` will pass through error\n * from source Observable immediately when it is emitted. In general it is highly recommended to use `delay` operator\n * for any kind of delaying of values in the stream, while using `observeOn` to specify which scheduler should be used\n * for notification emissions in general.\n *\n * ## Example\n *\n * Ensure values in subscribe are called just before browser repaint\n *\n * ```ts\n * import { interval, observeOn, animationFrameScheduler } from 'rxjs';\n *\n * const someDiv = document.createElement('div');\n * someDiv.style.cssText = 'width: 200px;background: #09c';\n * document.body.appendChild(someDiv);\n * const intervals = interval(10);      // Intervals are scheduled\n *                                      // with async scheduler by default...\n * intervals.pipe(\n *   observeOn(animationFrameScheduler) // ...but we will observe on animationFrame\n * )                                    // scheduler to ensure smooth animation.\n * .subscribe(val => {\n *   someDiv.style.height = val + 'px';\n * });\n * ```\n *\n * @see {@link delay}\n *\n * @param scheduler Scheduler that will be used to reschedule notifications from source Observable.\n * @param delay Number of milliseconds that states with what delay every notification should be rescheduled.\n * @return A function that returns an Observable that emits the same\n * notifications as the source Observable, but with provided scheduler.\n */\nexport function observeOn<T>(scheduler: SchedulerLike, delay = 0): MonoTypeOperatorFunction<T> {\n  return operate((source, subscriber) => {\n    source.subscribe(\n      createOperatorSubscriber(\n        subscriber,\n        (value) => executeSchedule(subscriber, scheduler, () => subscriber.next(value), delay),\n        () => executeSchedule(subscriber, scheduler, () => subscriber.complete(), delay),\n        (err) => executeSchedule(subscriber, scheduler, () => subscriber.error(err), delay)\n      )\n    );\n  });\n}\n", "import { MonoTypeOperatorFunction, SchedulerLike } from '../types';\nimport { operate } from '../util/lift';\n\n/**\n * Asynchronously subscribes Observers to this Observable on the specified {@link SchedulerLike}.\n *\n * With `subscribeOn` you can decide what type of scheduler a specific Observable will be using when it is subscribed to.\n *\n * Schedulers control the speed and order of emissions to observers from an Observable stream.\n *\n * ![](subscribeOn.png)\n *\n * ## Example\n *\n * Given the following code:\n *\n * ```ts\n * import { of, merge } from 'rxjs';\n *\n * const a = of(1, 2, 3);\n * const b = of(4, 5, 6);\n *\n * merge(a, b).subscribe(console.log);\n *\n * // Outputs\n * // 1\n * // 2\n * // 3\n * // 4\n * // 5\n * // 6\n * ```\n *\n * Both Observable `a` and `b` will emit their values directly and synchronously once they are subscribed to.\n *\n * If we instead use the `subscribeOn` operator declaring that we want to use the {@link asyncScheduler} for values emitted by Observable `a`:\n *\n * ```ts\n * import { of, subscribeOn, asyncScheduler, merge } from 'rxjs';\n *\n * const a = of(1, 2, 3).pipe(subscribeOn(asyncScheduler));\n * const b = of(4, 5, 6);\n *\n * merge(a, b).subscribe(console.log);\n *\n * // Outputs\n * // 4\n * // 5\n * // 6\n * // 1\n * // 2\n * // 3\n * ```\n *\n * The reason for this is that Observable `b` emits its values directly and synchronously like before\n * but the emissions from `a` are scheduled on the event loop because we are now using the {@link asyncScheduler} for that specific Observable.\n *\n * @param scheduler The {@link SchedulerLike} to perform subscription actions on.\n * @param delay A delay to pass to the scheduler to delay subscriptions\n * @return A function that returns an Observable modified so that its\n * subscriptions happen on the specified {@link SchedulerLike}.\n */\nexport function subscribeOn<T>(scheduler: SchedulerLike, delay: number = 0): MonoTypeOperatorFunction<T> {\n  return operate((source, subscriber) => {\n    subscriber.add(scheduler.schedule(() => source.subscribe(subscriber), delay));\n  });\n}\n", "import { innerFrom } from '../observable/innerFrom';\nimport { observeOn } from '../operators/observeOn';\nimport { subscribeOn } from '../operators/subscribeOn';\nimport { InteropObservable, SchedulerLike } from '../types';\n\nexport function scheduleObservable<T>(input: InteropObservable<T>, scheduler: SchedulerLike) {\n  return innerFrom(input).pipe(subscribeOn(scheduler), observeOn(scheduler));\n}\n", "import { innerFrom } from '../observable/innerFrom';\nimport { observeOn } from '../operators/observeOn';\nimport { subscribeOn } from '../operators/subscribeOn';\nimport { SchedulerLike } from '../types';\n\nexport function schedulePromise<T>(input: PromiseLike<T>, scheduler: SchedulerLike) {\n  return innerFrom(input).pipe(subscribeOn(scheduler), observeOn(scheduler));\n}\n", "import { Observable } from '../Observable';\nimport { SchedulerLike } from '../types';\n\nexport function scheduleArray<T>(input: ArrayLike<T>, scheduler: SchedulerLike) {\n  return new Observable<T>((subscriber) => {\n    // The current array index.\n    let i = 0;\n    // Start iterating over the array like on a schedule.\n    return scheduler.schedule(function () {\n      if (i === input.length) {\n        // If we have hit the end of the array like in the\n        // previous job, we can complete.\n        subscriber.complete();\n      } else {\n        // Otherwise let's next the value at the current index,\n        // then increment our index.\n        subscriber.next(input[i++]);\n        // If the last emission didn't cause us to close the subscriber\n        // (via take or some side effect), reschedule the job and we'll\n        // make another pass.\n        if (!subscriber.closed) {\n          this.schedule();\n        }\n      }\n    });\n  });\n}\n", "import { Observable } from '../Observable';\nimport { SchedulerLike } from '../types';\nimport { iterator as Symbol_iterator } from '../symbol/iterator';\nimport { isFunction } from '../util/isFunction';\nimport { executeSchedule } from '../util/executeSchedule';\n\n/**\n * Used in {@link scheduled} to create an observable from an Iterable.\n * @param input The iterable to create an observable from\n * @param scheduler The scheduler to use\n */\nexport function scheduleIterable<T>(input: Iterable<T>, scheduler: SchedulerLike) {\n  return new Observable<T>((subscriber) => {\n    let iterator: Iterator<T, T>;\n\n    // Schedule the initial creation of the iterator from\n    // the iterable. This is so the code in the iterable is\n    // not called until the scheduled job fires.\n    executeSchedule(subscriber, scheduler, () => {\n      // Create the iterator.\n      iterator = (input as any)[Symbol_iterator]();\n\n      executeSchedule(\n        subscriber,\n        scheduler,\n        () => {\n          let value: T;\n          let done: boolean | undefined;\n          try {\n            // Pull the value out of the iterator\n            ({ value, done } = iterator.next());\n          } catch (err) {\n            // We got an error while pulling from the iterator\n            subscriber.error(err);\n            return;\n          }\n\n          if (done) {\n            // If it is \"done\" we just complete. This mimics the\n            // behavior of JavaScript's `for..of` consumption of\n            // iterables, which will not emit the value from an iterator\n            // result of `{ done: true: value: 'here' }`.\n            subscriber.complete();\n          } else {\n            // The iterable is not done, emit the value.\n            subscriber.next(value);\n          }\n        },\n        0,\n        true\n      );\n    });\n\n    // During finalization, if we see this iterator has a `return` method,\n    // then we know it is a Generator, and not just an Iterator. So we call\n    // the `return()` function. This will ensure that any `finally { }` blocks\n    // inside of the generator we can hit will be hit properly.\n    return () => isFunction(iterator?.return) && iterator.return();\n  });\n}\n", "import { SchedulerLike } from '../types';\nimport { Observable } from '../Observable';\nimport { executeSchedule } from '../util/executeSchedule';\n\nexport function scheduleAsyncIterable<T>(input: AsyncIterable<T>, scheduler: SchedulerLike) {\n  if (!input) {\n    throw new Error('Iterable cannot be null');\n  }\n  return new Observable<T>((subscriber) => {\n    executeSchedule(subscriber, scheduler, () => {\n      const iterator = input[Symbol.asyncIterator]();\n      executeSchedule(\n        subscriber,\n        scheduler,\n        () => {\n          iterator.next().then((result) => {\n            if (result.done) {\n              // This will remove the subscriptions from\n              // the parent subscription.\n              subscriber.complete();\n            } else {\n              subscriber.next(result.value);\n            }\n          });\n        },\n        0,\n        true\n      );\n    });\n  });\n}\n", "import { SchedulerLike, ReadableStreamLike } from '../types';\nimport { Observable } from '../Observable';\nimport { scheduleAsyncIterable } from './scheduleAsyncIterable';\nimport { readableStreamLikeToAsyncGenerator } from '../util/isReadableStreamLike';\n\nexport function scheduleReadableStreamLike<T>(input: ReadableStreamLike<T>, scheduler: SchedulerLike): Observable<T> {\n  return scheduleAsyncIterable(readableStreamLikeToAsyncGenerator(input), scheduler);\n}\n", "import { scheduleObservable } from './scheduleObservable';\nimport { schedulePromise } from './schedulePromise';\nimport { scheduleArray } from './scheduleArray';\nimport { scheduleIterable } from './scheduleIterable';\nimport { scheduleAsyncIterable } from './scheduleAsyncIterable';\nimport { isInteropObservable } from '../util/isInteropObservable';\nimport { isPromise } from '../util/isPromise';\nimport { isArrayLike } from '../util/isArrayLike';\nimport { isIterable } from '../util/isIterable';\nimport { ObservableInput, SchedulerLike } from '../types';\nimport { Observable } from '../Observable';\nimport { isAsyncIterable } from '../util/isAsyncIterable';\nimport { createInvalidObservableTypeError } from '../util/throwUnobservableError';\nimport { isReadableStreamLike } from '../util/isReadableStreamLike';\nimport { scheduleReadableStreamLike } from './scheduleReadableStreamLike';\n\n/**\n * Converts from a common {@link ObservableInput} type to an observable where subscription and emissions\n * are scheduled on the provided scheduler.\n *\n * @see {@link from}\n * @see {@link of}\n *\n * @param input The observable, array, promise, iterable, etc you would like to schedule\n * @param scheduler The scheduler to use to schedule the subscription and emissions from\n * the returned observable.\n */\nexport function scheduled<T>(input: ObservableInput<T>, scheduler: SchedulerLike): Observable<T> {\n  if (input != null) {\n    if (isInteropObservable(input)) {\n      return scheduleObservable(input, scheduler);\n    }\n    if (isArrayLike(input)) {\n      return scheduleArray(input, scheduler);\n    }\n    if (isPromise(input)) {\n      return schedulePromise(input, scheduler);\n    }\n    if (isAsyncIterable(input)) {\n      return scheduleAsyncIterable(input, scheduler);\n    }\n    if (isIterable(input)) {\n      return scheduleIterable(input, scheduler);\n    }\n    if (isReadableStreamLike(input)) {\n      return scheduleReadableStreamLike(input, scheduler);\n    }\n  }\n  throw createInvalidObservableTypeError(input);\n}\n", "import { Observable } from '../Observable';\nimport { ObservableInput, SchedulerLike, ObservedValueOf } from '../types';\nimport { scheduled } from '../scheduled/scheduled';\nimport { innerFrom } from './innerFrom';\n\nexport function from<O extends ObservableInput<any>>(input: O): Observable<ObservedValueOf<O>>;\n/** @deprecated The `scheduler` parameter will be removed in v8. Use `scheduled`. Details: https://rxjs.dev/deprecations/scheduler-argument */\nexport function from<O extends ObservableInput<any>>(input: O, scheduler: SchedulerLike | undefined): Observable<ObservedValueOf<O>>;\n\n/**\n * Creates an Observable from an Array, an array-like object, a Promise, an iterable object, or an Observable-like object.\n *\n * <span class=\"informal\">Converts almost anything to an Observable.</span>\n *\n * ![](from.png)\n *\n * `from` converts various other objects and data types into Observables. It also converts a Promise, an array-like, or an\n * <a href=\"https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Iteration_protocols#iterable\" target=\"_blank\">iterable</a>\n * object into an Observable that emits the items in that promise, array, or iterable. A String, in this context, is treated\n * as an array of characters. Observable-like objects (contains a function named with the ES2015 Symbol for Observable) can also be\n * converted through this operator.\n *\n * ## Examples\n *\n * Converts an array to an Observable\n *\n * ```ts\n * import { from } from 'rxjs';\n *\n * const array = [10, 20, 30];\n * const result = from(array);\n *\n * result.subscribe(x => console.log(x));\n *\n * // Logs:\n * // 10\n * // 20\n * // 30\n * ```\n *\n * Convert an infinite iterable (from a generator) to an Observable\n *\n * ```ts\n * import { from, take } from 'rxjs';\n *\n * function* generateDoubles(seed) {\n *    let i = seed;\n *    while (true) {\n *      yield i;\n *      i = 2 * i; // double it\n *    }\n * }\n *\n * const iterator = generateDoubles(3);\n * const result = from(iterator).pipe(take(10));\n *\n * result.subscribe(x => console.log(x));\n *\n * // Logs:\n * // 3\n * // 6\n * // 12\n * // 24\n * // 48\n * // 96\n * // 192\n * // 384\n * // 768\n * // 1536\n * ```\n *\n * With `asyncScheduler`\n *\n * ```ts\n * import { from, asyncScheduler } from 'rxjs';\n *\n * console.log('start');\n *\n * const array = [10, 20, 30];\n * const result = from(array, asyncScheduler);\n *\n * result.subscribe(x => console.log(x));\n *\n * console.log('end');\n *\n * // Logs:\n * // 'start'\n * // 'end'\n * // 10\n * // 20\n * // 30\n * ```\n *\n * @see {@link fromEvent}\n * @see {@link fromEventPattern}\n *\n * @param {ObservableInput<T>} A subscription object, a Promise, an Observable-like,\n * an Array, an iterable, or an array-like object to be converted.\n * @param {SchedulerLike} An optional {@link SchedulerLike} on which to schedule the emission of values.\n * @return {Observable<T>}\n */\nexport function from<T>(input: ObservableInput<T>, scheduler?: SchedulerLike): Observable<T> {\n  return scheduler ? scheduled(input, scheduler) : innerFrom(input);\n}\n", "import { SchedulerLike, ValueFromArray } from '../types';\nimport { Observable } from '../Observable';\nimport { popScheduler } from '../util/args';\nimport { from } from './from';\n\n// Devs are more likely to pass null or undefined than they are a scheduler\n// without accompanying values. To make things easier for (naughty) devs who\n// use the `strictNullChecks: false` TypeScript compiler option, these\n// overloads with explicit null and undefined values are included.\n\nexport function of(value: null): Observable<null>;\nexport function of(value: undefined): Observable<undefined>;\n\n/** @deprecated The `scheduler` parameter will be removed in v8. Use `scheduled`. Details: https://rxjs.dev/deprecations/scheduler-argument */\nexport function of(scheduler: SchedulerLike): Observable<never>;\n/** @deprecated The `scheduler` parameter will be removed in v8. Use `scheduled`. Details: https://rxjs.dev/deprecations/scheduler-argument */\nexport function of<A extends readonly unknown[]>(...valuesAndScheduler: [...A, SchedulerLike]): Observable<ValueFromArray<A>>;\n\nexport function of(): Observable<never>;\n/** @deprecated Do not specify explicit type parameters. Signatures with type parameters that cannot be inferred will be removed in v8. */\nexport function of<T>(): Observable<T>;\nexport function of<T>(value: T): Observable<T>;\nexport function of<A extends readonly unknown[]>(...values: A): Observable<ValueFromArray<A>>;\n\n/**\n * Converts the arguments to an observable sequence.\n *\n * <span class=\"informal\">Each argument becomes a `next` notification.</span>\n *\n * ![](of.png)\n *\n * Unlike {@link from}, it does not do any flattening and emits each argument in whole\n * as a separate `next` notification.\n *\n * ## Examples\n *\n * Emit the values `10, 20, 30`\n *\n * ```ts\n * import { of } from 'rxjs';\n *\n * of(10, 20, 30)\n *   .subscribe({\n *     next: value => console.log('next:', value),\n *     error: err => console.log('error:', err),\n *     complete: () => console.log('the end'),\n *   });\n *\n * // Outputs\n * // next: 10\n * // next: 20\n * // next: 30\n * // the end\n * ```\n *\n * Emit the array `[1, 2, 3]`\n *\n * ```ts\n * import { of } from 'rxjs';\n *\n * of([1, 2, 3])\n *   .subscribe({\n *     next: value => console.log('next:', value),\n *     error: err => console.log('error:', err),\n *     complete: () => console.log('the end'),\n *   });\n *\n * // Outputs\n * // next: [1, 2, 3]\n * // the end\n * ```\n *\n * @see {@link from}\n * @see {@link range}\n *\n * @param {...T} values A comma separated list of arguments you want to be emitted\n * @return {Observable} An Observable that emits the arguments\n * described above and then completes.\n */\nexport function of<T>(...args: Array<T | SchedulerLike>): Observable<T> {\n  const scheduler = popScheduler(args);\n  return from(args as T[], scheduler);\n}\n", "import { Observable } from '../Observable';\nimport { Subscriber } from '../Subscriber';\nimport { SchedulerLike } from '../types';\nimport { isFunction } from '../util/isFunction';\n\n/**\n * Creates an observable that will create an error instance and push it to the consumer as an error\n * immediately upon subscription.\n *\n * <span class=\"informal\">Just errors and does nothing else</span>\n *\n * ![](throw.png)\n *\n * This creation function is useful for creating an observable that will create an error and error every\n * time it is subscribed to. Generally, inside of most operators when you might want to return an errored\n * observable, this is unnecessary. In most cases, such as in the inner return of {@link concatMap},\n * {@link mergeMap}, {@link defer}, and many others, you can simply throw the error, and RxJS will pick\n * that up and notify the consumer of the error.\n *\n * ## Example\n *\n * Create a simple observable that will create a new error with a timestamp and log it\n * and the message every time you subscribe to it\n *\n * ```ts\n * import { throwError } from 'rxjs';\n *\n * let errorCount = 0;\n *\n * const errorWithTimestamp$ = throwError(() => {\n *   const error: any = new Error(`This is error number ${ ++errorCount }`);\n *   error.timestamp = Date.now();\n *   return error;\n * });\n *\n * errorWithTimestamp$.subscribe({\n *   error: err => console.log(err.timestamp, err.message)\n * });\n *\n * errorWithTimestamp$.subscribe({\n *   error: err => console.log(err.timestamp, err.message)\n * });\n *\n * // Logs the timestamp and a new error message for each subscription\n * ```\n *\n * ### Unnecessary usage\n *\n * Using `throwError` inside of an operator or creation function\n * with a callback, is usually not necessary\n *\n * ```ts\n * import { of, concatMap, timer, throwError } from 'rxjs';\n *\n * const delays$ = of(1000, 2000, Infinity, 3000);\n *\n * delays$.pipe(\n *   concatMap(ms => {\n *     if (ms < 10000) {\n *       return timer(ms);\n *     } else {\n *       // This is probably overkill.\n *       return throwError(() => new Error(`Invalid time ${ ms }`));\n *     }\n *   })\n * )\n * .subscribe({\n *   next: console.log,\n *   error: console.error\n * });\n * ```\n *\n * You can just throw the error instead\n *\n * ```ts\n * import { of, concatMap, timer } from 'rxjs';\n *\n * const delays$ = of(1000, 2000, Infinity, 3000);\n *\n * delays$.pipe(\n *   concatMap(ms => {\n *     if (ms < 10000) {\n *       return timer(ms);\n *     } else {\n *       // Cleaner and easier to read for most folks.\n *       throw new Error(`Invalid time ${ ms }`);\n *     }\n *   })\n * )\n * .subscribe({\n *   next: console.log,\n *   error: console.error\n * });\n * ```\n *\n * @param errorFactory A factory function that will create the error instance that is pushed.\n */\nexport function throwError(errorFactory: () => any): Observable<never>;\n\n/**\n * Returns an observable that will error with the specified error immediately upon subscription.\n *\n * @param error The error instance to emit\n * @deprecated Support for passing an error value will be removed in v8. Instead, pass a factory function to `throwError(() => new Error('test'))`. This is\n * because it will create the error at the moment it should be created and capture a more appropriate stack trace. If\n * for some reason you need to create the error ahead of time, you can still do that: `const err = new Error('test'); throwError(() => err);`.\n */\nexport function throwError(error: any): Observable<never>;\n\n/**\n * Notifies the consumer of an error using a given scheduler by scheduling it at delay `0` upon subscription.\n *\n * @param errorOrErrorFactory An error instance or error factory\n * @param scheduler A scheduler to use to schedule the error notification\n * @deprecated The `scheduler` parameter will be removed in v8.\n * Use `throwError` in combination with {@link observeOn}: `throwError(() => new Error('test')).pipe(observeOn(scheduler));`.\n * Details: https://rxjs.dev/deprecations/scheduler-argument\n */\nexport function throwError(errorOrErrorFactory: any, scheduler: SchedulerLike): Observable<never>;\n\nexport function throwError(errorOrErrorFactory: any, scheduler?: SchedulerLike): Observable<never> {\n  const errorFactory = isFunction(errorOrErrorFactory) ? errorOrErrorFactory : () => errorOrErrorFactory;\n  const init = (subscriber: Subscriber<never>) => subscriber.error(errorFactory());\n  return new Observable(scheduler ? (subscriber) => scheduler.schedule(init as any, 0, subscriber) : init);\n}\n", "import { createErrorClass } from './createErrorClass';\n\nexport interface EmptyError extends Error {}\n\nexport interface EmptyErrorCtor {\n  /**\n   * @deprecated Internal implementation detail. Do not construct error instances.\n   * Cannot be tagged as internal: https://github.com/ReactiveX/rxjs/issues/6269\n   */\n  new (): EmptyError;\n}\n\n/**\n * An error thrown when an Observable or a sequence was queried but has no\n * elements.\n *\n * @see {@link first}\n * @see {@link last}\n * @see {@link single}\n * @see {@link firstValueFrom}\n * @see {@link lastValueFrom}\n *\n * @class EmptyError\n */\nexport const EmptyError: EmptyErrorCtor = createErrorClass((_super) => function EmptyErrorImpl(this: any) {\n  _super(this);\n  this.name = 'EmptyError';\n  this.message = 'no elements in sequence';\n});\n", "/**\n * Checks to see if a value is not only a `Date` object,\n * but a *valid* `Date` object that can be converted to a\n * number. For example, `new Date('blah')` is indeed an\n * `instanceof Date`, however it cannot be converted to a\n * number.\n */\nexport function isValidDate(value: any): value is Date {\n  return value instanceof Date && !isNaN(value as any);\n}\n", "import { OperatorFunction } from '../types';\nimport { operate } from '../util/lift';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\n\nexport function map<T, R>(project: (value: T, index: number) => R): OperatorFunction<T, R>;\n/** @deprecated Use a closure instead of a `thisArg`. Signatures accepting a `thisArg` will be removed in v8. */\nexport function map<T, R, A>(project: (this: A, value: T, index: number) => R, thisArg: A): OperatorFunction<T, R>;\n\n/**\n * Applies a given `project` function to each value emitted by the source\n * Observable, and emits the resulting values as an Observable.\n *\n * <span class=\"informal\">Like [Array.prototype.map()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/map),\n * it passes each source value through a transformation function to get\n * corresponding output values.</span>\n *\n * ![](map.png)\n *\n * Similar to the well known `Array.prototype.map` function, this operator\n * applies a projection to each value and emits that projection in the output\n * Observable.\n *\n * ## Example\n *\n * Map every click to the `clientX` position of that click\n *\n * ```ts\n * import { fromEvent, map } from 'rxjs';\n *\n * const clicks = fromEvent<PointerEvent>(document, 'click');\n * const positions = clicks.pipe(map(ev => ev.clientX));\n *\n * positions.subscribe(x => console.log(x));\n * ```\n *\n * @see {@link mapTo}\n * @see {@link pluck}\n *\n * @param {function(value: T, index: number): R} project The function to apply\n * to each `value` emitted by the source Observable. The `index` parameter is\n * the number `i` for the i-th emission that has happened since the\n * subscription, starting from the number `0`.\n * @param {any} [thisArg] An optional argument to define what `this` is in the\n * `project` function.\n * @return A function that returns an Observable that emits the values from the\n * source Observable transformed by the given `project` function.\n */\nexport function map<T, R>(project: (value: T, index: number) => R, thisArg?: any): OperatorFunction<T, R> {\n  return operate((source, subscriber) => {\n    // The index of the value from the source. Used with projection.\n    let index = 0;\n    // Subscribe to the source, all errors and completions are sent along\n    // to the consumer.\n    source.subscribe(\n      createOperatorSubscriber(subscriber, (value: T) => {\n        // Call the projection function with the appropriate this context,\n        // and send the resulting value to the consumer.\n        subscriber.next(project.call(thisArg, value, index++));\n      })\n    );\n  });\n}\n", "import { OperatorFunction } from \"../types\";\nimport { map } from \"../operators/map\";\n\nconst { isArray } = Array;\n\nfunction callOrApply<T, R>(fn: ((...values: T[]) => R), args: T|T[]): R {\n    return isArray(args) ? fn(...args) : fn(args);\n}\n\n/**\n * Used in several -- mostly deprecated -- situations where we need to \n * apply a list of arguments or a single argument to a result selector.\n */\nexport function mapOneOrManyArgs<T, R>(fn: ((...values: T[]) => R)): OperatorFunction<T|T[], R> {\n    return map(args => callOrApply(fn, args))\n}", "const { isArray } = Array;\nconst { getPrototypeOf, prototype: objectProto, keys: getKeys } = Object;\n\n/**\n * Used in functions where either a list of arguments, a single array of arguments, or a\n * dictionary of arguments can be returned. Returns an object with an `args` property with\n * the arguments in an array, if it is a dictionary, it will also return the `keys` in another\n * property.\n */\nexport function argsArgArrayOrObject<T, O extends Record<string, T>>(args: T[] | [O] | [T[]]): { args: T[]; keys: string[] | null } {\n  if (args.length === 1) {\n    const first = args[0];\n    if (isArray(first)) {\n      return { args: first, keys: null };\n    }\n    if (isPOJO(first)) {\n      const keys = getKeys(first);\n      return {\n        args: keys.map((key) => first[key]),\n        keys,\n      };\n    }\n  }\n\n  return { args: args as T[], keys: null };\n}\n\nfunction isPOJO(obj: any): obj is object {\n  return obj && typeof obj === 'object' && getPrototypeOf(obj) === objectProto;\n}\n", "export function createObject(keys: string[], values: any[]) {\n  return keys.reduce((result, key, i) => ((result[key] = values[i]), result), {} as any);\n}\n", "import { Observable } from '../Observable';\nimport { ObservableInput, SchedulerLike, ObservedValueOf, ObservableInputTuple } from '../types';\nimport { argsArgArrayOrObject } from '../util/argsArgArrayOrObject';\nimport { Subscriber } from '../Subscriber';\nimport { from } from './from';\nimport { identity } from '../util/identity';\nimport { Subscription } from '../Subscription';\nimport { mapOneOrManyArgs } from '../util/mapOneOrManyArgs';\nimport { popResultSelector, popScheduler } from '../util/args';\nimport { createObject } from '../util/createObject';\nimport { createOperatorSubscriber } from '../operators/OperatorSubscriber';\nimport { AnyCatcher } from '../AnyCatcher';\nimport { executeSchedule } from '../util/executeSchedule';\n\n// combineLatest(any)\n// We put this first because we need to catch cases where the user has supplied\n// _exactly `any`_ as the argument. Since `any` literally matches _anything_,\n// we don't want it to randomly hit one of the other type signatures below,\n// as we have no idea at build-time what type we should be returning when given an any.\n\n/**\n * You have passed `any` here, we can't figure out if it is\n * an array or an object, so you're getting `unknown`. Use better types.\n * @param arg Something typed as `any`\n */\nexport function combineLatest<T extends AnyCatcher>(arg: T): Observable<unknown>;\n\n// combineLatest([a, b, c])\nexport function combineLatest(sources: []): Observable<never>;\nexport function combineLatest<A extends readonly unknown[]>(sources: readonly [...ObservableInputTuple<A>]): Observable<A>;\n/** @deprecated The `scheduler` parameter will be removed in v8. Use `scheduled` and `combineLatestAll`. Details: https://rxjs.dev/deprecations/scheduler-argument */\nexport function combineLatest<A extends readonly unknown[], R>(\n  sources: readonly [...ObservableInputTuple<A>],\n  resultSelector: (...values: A) => R,\n  scheduler: SchedulerLike\n): Observable<R>;\nexport function combineLatest<A extends readonly unknown[], R>(\n  sources: readonly [...ObservableInputTuple<A>],\n  resultSelector: (...values: A) => R\n): Observable<R>;\n/** @deprecated The `scheduler` parameter will be removed in v8. Use `scheduled` and `combineLatestAll`. Details: https://rxjs.dev/deprecations/scheduler-argument */\nexport function combineLatest<A extends readonly unknown[]>(\n  sources: readonly [...ObservableInputTuple<A>],\n  scheduler: SchedulerLike\n): Observable<A>;\n\n// combineLatest(a, b, c)\n/** @deprecated Pass an array of sources instead. The rest-parameters signature will be removed in v8. Details: https://rxjs.dev/deprecations/array-argument */\nexport function combineLatest<A extends readonly unknown[]>(...sources: [...ObservableInputTuple<A>]): Observable<A>;\n/** @deprecated The `scheduler` parameter will be removed in v8. Use `scheduled` and `combineLatestAll`. Details: https://rxjs.dev/deprecations/scheduler-argument */\nexport function combineLatest<A extends readonly unknown[], R>(\n  ...sourcesAndResultSelectorAndScheduler: [...ObservableInputTuple<A>, (...values: A) => R, SchedulerLike]\n): Observable<R>;\n/** @deprecated Pass an array of sources instead. The rest-parameters signature will be removed in v8. Details: https://rxjs.dev/deprecations/array-argument */\nexport function combineLatest<A extends readonly unknown[], R>(\n  ...sourcesAndResultSelector: [...ObservableInputTuple<A>, (...values: A) => R]\n): Observable<R>;\n/** @deprecated The `scheduler` parameter will be removed in v8. Use `scheduled` and `combineLatestAll`. Details: https://rxjs.dev/deprecations/scheduler-argument */\nexport function combineLatest<A extends readonly unknown[]>(\n  ...sourcesAndScheduler: [...ObservableInputTuple<A>, SchedulerLike]\n): Observable<A>;\n\n// combineLatest({a, b, c})\nexport function combineLatest(sourcesObject: { [K in any]: never }): Observable<never>;\nexport function combineLatest<T extends Record<string, ObservableInput<any>>>(\n  sourcesObject: T\n): Observable<{ [K in keyof T]: ObservedValueOf<T[K]> }>;\n\n/**\n * Combines multiple Observables to create an Observable whose values are\n * calculated from the latest values of each of its input Observables.\n *\n * <span class=\"informal\">Whenever any input Observable emits a value, it\n * computes a formula using the latest values from all the inputs, then emits\n * the output of that formula.</span>\n *\n * ![](combineLatest.png)\n *\n * `combineLatest` combines the values from all the Observables passed in the\n * observables array. This is done by subscribing to each Observable in order and,\n * whenever any Observable emits, collecting an array of the most recent\n * values from each Observable. So if you pass `n` Observables to this operator,\n * the returned Observable will always emit an array of `n` values, in an order\n * corresponding to the order of the passed Observables (the value from the first Observable\n * will be at index 0 of the array and so on).\n *\n * Static version of `combineLatest` accepts an array of Observables. Note that an array of\n * Observables is a good choice, if you don't know beforehand how many Observables\n * you will combine. Passing an empty array will result in an Observable that\n * completes immediately.\n *\n * To ensure the output array always has the same length, `combineLatest` will\n * actually wait for all input Observables to emit at least once,\n * before it starts emitting results. This means if some Observable emits\n * values before other Observables started emitting, all these values but the last\n * will be lost. On the other hand, if some Observable does not emit a value but\n * completes, resulting Observable will complete at the same moment without\n * emitting anything, since it will now be impossible to include a value from the\n * completed Observable in the resulting array. Also, if some input Observable does\n * not emit any value and never completes, `combineLatest` will also never emit\n * and never complete, since, again, it will wait for all streams to emit some\n * value.\n *\n * If at least one Observable was passed to `combineLatest` and all passed Observables\n * emitted something, the resulting Observable will complete when all combined\n * streams complete. So even if some Observable completes, the result of\n * `combineLatest` will still emit values when other Observables do. In case\n * of a completed Observable, its value from now on will always be the last\n * emitted value. On the other hand, if any Observable errors, `combineLatest`\n * will error immediately as well, and all other Observables will be unsubscribed.\n *\n * ## Examples\n *\n * Combine two timer Observables\n *\n * ```ts\n * import { timer, combineLatest } from 'rxjs';\n *\n * const firstTimer = timer(0, 1000); // emit 0, 1, 2... after every second, starting from now\n * const secondTimer = timer(500, 1000); // emit 0, 1, 2... after every second, starting 0,5s from now\n * const combinedTimers = combineLatest([firstTimer, secondTimer]);\n * combinedTimers.subscribe(value => console.log(value));\n * // Logs\n * // [0, 0] after 0.5s\n * // [1, 0] after 1s\n * // [1, 1] after 1.5s\n * // [2, 1] after 2s\n * ```\n *\n * Combine a dictionary of Observables\n *\n * ```ts\n * import { of, delay, startWith, combineLatest } from 'rxjs';\n *\n * const observables = {\n *   a: of(1).pipe(delay(1000), startWith(0)),\n *   b: of(5).pipe(delay(5000), startWith(0)),\n *   c: of(10).pipe(delay(10000), startWith(0))\n * };\n * const combined = combineLatest(observables);\n * combined.subscribe(value => console.log(value));\n * // Logs\n * // { a: 0, b: 0, c: 0 } immediately\n * // { a: 1, b: 0, c: 0 } after 1s\n * // { a: 1, b: 5, c: 0 } after 5s\n * // { a: 1, b: 5, c: 10 } after 10s\n * ```\n *\n * Combine an array of Observables\n *\n * ```ts\n * import { of, delay, startWith, combineLatest } from 'rxjs';\n *\n * const observables = [1, 5, 10].map(\n *   n => of(n).pipe(\n *     delay(n * 1000), // emit 0 and then emit n after n seconds\n *     startWith(0)\n *   )\n * );\n * const combined = combineLatest(observables);\n * combined.subscribe(value => console.log(value));\n * // Logs\n * // [0, 0, 0] immediately\n * // [1, 0, 0] after 1s\n * // [1, 5, 0] after 5s\n * // [1, 5, 10] after 10s\n * ```\n *\n * Use map operator to dynamically calculate the Body-Mass Index\n *\n * ```ts\n * import { of, combineLatest, map } from 'rxjs';\n *\n * const weight = of(70, 72, 76, 79, 75);\n * const height = of(1.76, 1.77, 1.78);\n * const bmi = combineLatest([weight, height]).pipe(\n *   map(([w, h]) => w / (h * h)),\n * );\n * bmi.subscribe(x => console.log('BMI is ' + x));\n *\n * // With output to console:\n * // BMI is 24.212293388429753\n * // BMI is 23.93948099205209\n * // BMI is 23.671253629592222\n * ```\n *\n * @see {@link combineLatestAll}\n * @see {@link merge}\n * @see {@link withLatestFrom}\n *\n * @param {ObservableInput} [observables] An array of input Observables to combine with each other.\n * An array of Observables must be given as the first argument.\n * @param {function} [project] An optional function to project the values from\n * the combined latest values into a new value on the output Observable.\n * @param {SchedulerLike} [scheduler=null] The {@link SchedulerLike} to use for subscribing to\n * each input Observable.\n * @return {Observable} An Observable of projected values from the most recent\n * values from each input Observable, or an array of the most recent values from\n * each input Observable.\n */\nexport function combineLatest<O extends ObservableInput<any>, R>(...args: any[]): Observable<R> | Observable<ObservedValueOf<O>[]> {\n  const scheduler = popScheduler(args);\n  const resultSelector = popResultSelector(args);\n\n  const { args: observables, keys } = argsArgArrayOrObject(args);\n\n  if (observables.length === 0) {\n    // If no observables are passed, or someone has passed an empty array\n    // of observables, or even an empty object POJO, we need to just\n    // complete (EMPTY), but we have to honor the scheduler provided if any.\n    return from([], scheduler as any);\n  }\n\n  const result = new Observable<ObservedValueOf<O>[]>(\n    combineLatestInit(\n      observables as ObservableInput<ObservedValueOf<O>>[],\n      scheduler,\n      keys\n        ? // A handler for scrubbing the array of args into a dictionary.\n          (values) => createObject(keys, values)\n        : // A passthrough to just return the array\n          identity\n    )\n  );\n\n  return resultSelector ? (result.pipe(mapOneOrManyArgs(resultSelector)) as Observable<R>) : result;\n}\n\nexport function combineLatestInit(\n  observables: ObservableInput<any>[],\n  scheduler?: SchedulerLike,\n  valueTransform: (values: any[]) => any = identity\n) {\n  return (subscriber: Subscriber<any>) => {\n    // The outer subscription. We're capturing this in a function\n    // because we may have to schedule it.\n    maybeSchedule(\n      scheduler,\n      () => {\n        const { length } = observables;\n        // A store for the values each observable has emitted so far. We match observable to value on index.\n        const values = new Array(length);\n        // The number of currently active subscriptions, as they complete, we decrement this number to see if\n        // we are all done combining values, so we can complete the result.\n        let active = length;\n        // The number of inner sources that still haven't emitted the first value\n        // We need to track this because all sources need to emit one value in order\n        // to start emitting values.\n        let remainingFirstValues = length;\n        // The loop to kick off subscription. We're keying everything on index `i` to relate the observables passed\n        // in to the slot in the output array or the key in the array of keys in the output dictionary.\n        for (let i = 0; i < length; i++) {\n          maybeSchedule(\n            scheduler,\n            () => {\n              const source = from(observables[i], scheduler as any);\n              let hasFirstValue = false;\n              source.subscribe(\n                createOperatorSubscriber(\n                  subscriber,\n                  (value) => {\n                    // When we get a value, record it in our set of values.\n                    values[i] = value;\n                    if (!hasFirstValue) {\n                      // If this is our first value, record that.\n                      hasFirstValue = true;\n                      remainingFirstValues--;\n                    }\n                    if (!remainingFirstValues) {\n                      // We're not waiting for any more\n                      // first values, so we can emit!\n                      subscriber.next(valueTransform(values.slice()));\n                    }\n                  },\n                  () => {\n                    if (!--active) {\n                      // We only complete the result if we have no more active\n                      // inner observables.\n                      subscriber.complete();\n                    }\n                  }\n                )\n              );\n            },\n            subscriber\n          );\n        }\n      },\n      subscriber\n    );\n  };\n}\n\n/**\n * A small utility to handle the couple of locations where we want to schedule if a scheduler was provided,\n * but we don't if there was no scheduler.\n */\nfunction maybeSchedule(scheduler: SchedulerLike | undefined, execute: () => void, subscription: Subscription) {\n  if (scheduler) {\n    executeSchedule(subscription, scheduler, execute);\n  } else {\n    execute();\n  }\n}\n", "import { Observable } from '../Observable';\nimport { innerFrom } from '../observable/innerFrom';\nimport { Subscriber } from '../Subscriber';\nimport { ObservableInput, SchedulerLike } from '../types';\nimport { executeSchedule } from '../util/executeSchedule';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\n\n/**\n * A process embodying the general \"merge\" strategy. This is used in\n * `mergeMap` and `mergeScan` because the logic is otherwise nearly identical.\n * @param source The original source observable\n * @param subscriber The consumer subscriber\n * @param project The projection function to get our inner sources\n * @param concurrent The number of concurrent inner subscriptions\n * @param onBeforeNext Additional logic to apply before nexting to our consumer\n * @param expand If `true` this will perform an \"expand\" strategy, which differs only\n * in that it recurses, and the inner subscription must be schedule-able.\n * @param innerSubScheduler A scheduler to use to schedule inner subscriptions,\n * this is to support the expand strategy, mostly, and should be deprecated\n */\nexport function mergeInternals<T, R>(\n  source: Observable<T>,\n  subscriber: Subscriber<R>,\n  project: (value: T, index: number) => ObservableInput<R>,\n  concurrent: number,\n  onBeforeNext?: (innerValue: R) => void,\n  expand?: boolean,\n  innerSubScheduler?: SchedulerLike,\n  additionalFinalizer?: () => void\n) {\n  // Buffered values, in the event of going over our concurrency limit\n  const buffer: T[] = [];\n  // The number of active inner subscriptions.\n  let active = 0;\n  // An index to pass to our accumulator function\n  let index = 0;\n  // Whether or not the outer source has completed.\n  let isComplete = false;\n\n  /**\n   * Checks to see if we can complete our result or not.\n   */\n  const checkComplete = () => {\n    // If the outer has completed, and nothing is left in the buffer,\n    // and we don't have any active inner subscriptions, then we can\n    // Emit the state and complete.\n    if (isComplete && !buffer.length && !active) {\n      subscriber.complete();\n    }\n  };\n\n  // If we're under our concurrency limit, just start the inner subscription, otherwise buffer and wait.\n  const outerNext = (value: T) => (active < concurrent ? doInnerSub(value) : buffer.push(value));\n\n  const doInnerSub = (value: T) => {\n    // If we're expanding, we need to emit the outer values and the inner values\n    // as the inners will \"become outers\" in a way as they are recursively fed\n    // back to the projection mechanism.\n    expand && subscriber.next(value as any);\n\n    // Increment the number of active subscriptions so we can track it\n    // against our concurrency limit later.\n    active++;\n\n    // A flag used to show that the inner observable completed.\n    // This is checked during finalization to see if we should\n    // move to the next item in the buffer, if there is on.\n    let innerComplete = false;\n\n    // Start our inner subscription.\n    innerFrom(project(value, index++)).subscribe(\n      createOperatorSubscriber(\n        subscriber,\n        (innerValue) => {\n          // `mergeScan` has additional handling here. For example\n          // taking the inner value and updating state.\n          onBeforeNext?.(innerValue);\n\n          if (expand) {\n            // If we're expanding, then just recurse back to our outer\n            // handler. It will emit the value first thing.\n            outerNext(innerValue as any);\n          } else {\n            // Otherwise, emit the inner value.\n            subscriber.next(innerValue);\n          }\n        },\n        () => {\n          // Flag that we have completed, so we know to check the buffer\n          // during finalization.\n          innerComplete = true;\n        },\n        // Errors are passed to the destination.\n        undefined,\n        () => {\n          // During finalization, if the inner completed (it wasn't errored or\n          // cancelled), then we want to try the next item in the buffer if\n          // there is one.\n          if (innerComplete) {\n            // We have to wrap this in a try/catch because it happens during\n            // finalization, possibly asynchronously, and we want to pass\n            // any errors that happen (like in a projection function) to\n            // the outer Subscriber.\n            try {\n              // INNER SOURCE COMPLETE\n              // Decrement the active count to ensure that the next time\n              // we try to call `doInnerSub`, the number is accurate.\n              active--;\n              // If we have more values in the buffer, try to process those\n              // Note that this call will increment `active` ahead of the\n              // next conditional, if there were any more inner subscriptions\n              // to start.\n              while (buffer.length && active < concurrent) {\n                const bufferedValue = buffer.shift()!;\n                // Particularly for `expand`, we need to check to see if a scheduler was provided\n                // for when we want to start our inner subscription. Otherwise, we just start\n                // are next inner subscription.\n                if (innerSubScheduler) {\n                  executeSchedule(subscriber, innerSubScheduler, () => doInnerSub(bufferedValue));\n                } else {\n                  doInnerSub(bufferedValue);\n                }\n              }\n              // Check to see if we can complete, and complete if so.\n              checkComplete();\n            } catch (err) {\n              subscriber.error(err);\n            }\n          }\n        }\n      )\n    );\n  };\n\n  // Subscribe to our source observable.\n  source.subscribe(\n    createOperatorSubscriber(subscriber, outerNext, () => {\n      // Outer completed, make a note of it, and check to see if we can complete everything.\n      isComplete = true;\n      checkComplete();\n    })\n  );\n\n  // Additional finalization (for when the destination is torn down).\n  // Other finalization is added implicitly via subscription above.\n  return () => {\n    additionalFinalizer?.();\n  };\n}\n", "import { ObservableInput, OperatorFunction, ObservedValueOf } from '../types';\nimport { map } from './map';\nimport { innerFrom } from '../observable/innerFrom';\nimport { operate } from '../util/lift';\nimport { mergeInternals } from './mergeInternals';\nimport { isFunction } from '../util/isFunction';\n\n/* tslint:disable:max-line-length */\nexport function mergeMap<T, O extends ObservableInput<any>>(\n  project: (value: T, index: number) => O,\n  concurrent?: number\n): OperatorFunction<T, ObservedValueOf<O>>;\n/** @deprecated The `resultSelector` parameter will be removed in v8. Use an inner `map` instead. Details: https://rxjs.dev/deprecations/resultSelector */\nexport function mergeMap<T, O extends ObservableInput<any>>(\n  project: (value: T, index: number) => O,\n  resultSelector: undefined,\n  concurrent?: number\n): OperatorFunction<T, ObservedValueOf<O>>;\n/** @deprecated The `resultSelector` parameter will be removed in v8. Use an inner `map` instead. Details: https://rxjs.dev/deprecations/resultSelector */\nexport function mergeMap<T, R, O extends ObservableInput<any>>(\n  project: (value: T, index: number) => O,\n  resultSelector: (outerValue: T, innerValue: ObservedValueOf<O>, outerIndex: number, innerIndex: number) => R,\n  concurrent?: number\n): OperatorFunction<T, R>;\n/* tslint:enable:max-line-length */\n\n/**\n * Projects each source value to an Observable which is merged in the output\n * Observable.\n *\n * <span class=\"informal\">Maps each value to an Observable, then flattens all of\n * these inner Observables using {@link mergeAll}.</span>\n *\n * ![](mergeMap.png)\n *\n * Returns an Observable that emits items based on applying a function that you\n * supply to each item emitted by the source Observable, where that function\n * returns an Observable, and then merging those resulting Observables and\n * emitting the results of this merger.\n *\n * ## Example\n *\n * Map and flatten each letter to an Observable ticking every 1 second\n *\n * ```ts\n * import { of, mergeMap, interval, map } from 'rxjs';\n *\n * const letters = of('a', 'b', 'c');\n * const result = letters.pipe(\n *   mergeMap(x => interval(1000).pipe(map(i => x + i)))\n * );\n *\n * result.subscribe(x => console.log(x));\n *\n * // Results in the following:\n * // a0\n * // b0\n * // c0\n * // a1\n * // b1\n * // c1\n * // continues to list a, b, c every second with respective ascending integers\n * ```\n *\n * @see {@link concatMap}\n * @see {@link exhaustMap}\n * @see {@link merge}\n * @see {@link mergeAll}\n * @see {@link mergeMapTo}\n * @see {@link mergeScan}\n * @see {@link switchMap}\n *\n * @param {function(value: T, ?index: number): ObservableInput} project A function\n * that, when applied to an item emitted by the source Observable, returns an\n * Observable.\n * @param {number} [concurrent=Infinity] Maximum number of input\n * Observables being subscribed to concurrently.\n * @return A function that returns an Observable that emits the result of\n * applying the projection function (and the optional deprecated\n * `resultSelector`) to each item emitted by the source Observable and merging\n * the results of the Observables obtained from this transformation.\n */\nexport function mergeMap<T, R, O extends ObservableInput<any>>(\n  project: (value: T, index: number) => O,\n  resultSelector?: ((outerValue: T, innerValue: ObservedValueOf<O>, outerIndex: number, innerIndex: number) => R) | number,\n  concurrent: number = Infinity\n): OperatorFunction<T, ObservedValueOf<O> | R> {\n  if (isFunction(resultSelector)) {\n    // DEPRECATED PATH\n    return mergeMap((a, i) => map((b: any, ii: number) => resultSelector(a, b, i, ii))(innerFrom(project(a, i))), concurrent);\n  } else if (typeof resultSelector === 'number') {\n    concurrent = resultSelector;\n  }\n\n  return operate((source, subscriber) => mergeInternals(source, subscriber, project, concurrent));\n}\n", "import { mergeMap } from './mergeMap';\nimport { identity } from '../util/identity';\nimport { OperatorFunction, ObservableInput, ObservedValueOf } from '../types';\n\n/**\n * Converts a higher-order Observable into a first-order Observable which\n * concurrently delivers all values that are emitted on the inner Observables.\n *\n * <span class=\"informal\">Flattens an Observable-of-Observables.</span>\n *\n * ![](mergeAll.png)\n *\n * `mergeAll` subscribes to an Observable that emits Observables, also known as\n * a higher-order Observable. Each time it observes one of these emitted inner\n * Observables, it subscribes to that and delivers all the values from the\n * inner Observable on the output Observable. The output Observable only\n * completes once all inner Observables have completed. Any error delivered by\n * a inner Observable will be immediately emitted on the output Observable.\n *\n * ## Examples\n *\n * Spawn a new interval Observable for each click event, and blend their outputs as one Observable\n *\n * ```ts\n * import { fromEvent, map, interval, mergeAll } from 'rxjs';\n *\n * const clicks = fromEvent(document, 'click');\n * const higherOrder = clicks.pipe(map(() => interval(1000)));\n * const firstOrder = higherOrder.pipe(mergeAll());\n *\n * firstOrder.subscribe(x => console.log(x));\n * ```\n *\n * Count from 0 to 9 every second for each click, but only allow 2 concurrent timers\n *\n * ```ts\n * import { fromEvent, map, interval, take, mergeAll } from 'rxjs';\n *\n * const clicks = fromEvent(document, 'click');\n * const higherOrder = clicks.pipe(\n *   map(() => interval(1000).pipe(take(10)))\n * );\n * const firstOrder = higherOrder.pipe(mergeAll(2));\n *\n * firstOrder.subscribe(x => console.log(x));\n * ```\n *\n * @see {@link combineLatestAll}\n * @see {@link concatAll}\n * @see {@link exhaustAll}\n * @see {@link merge}\n * @see {@link mergeMap}\n * @see {@link mergeMapTo}\n * @see {@link mergeScan}\n * @see {@link switchAll}\n * @see {@link switchMap}\n * @see {@link zipAll}\n *\n * @param {number} [concurrent=Infinity] Maximum number of inner\n * Observables being subscribed to concurrently.\n * @return A function that returns an Observable that emits values coming from\n * all the inner Observables emitted by the source Observable.\n */\nexport function mergeAll<O extends ObservableInput<any>>(concurrent: number = Infinity): OperatorFunction<O, ObservedValueOf<O>> {\n  return mergeMap(identity, concurrent);\n}\n", "import { mergeAll } from './mergeAll';\nimport { OperatorFunction, ObservableInput, ObservedValueOf } from '../types';\n\n/**\n * Converts a higher-order Observable into a first-order Observable by\n * concatenating the inner Observables in order.\n *\n * <span class=\"informal\">Flattens an Observable-of-Observables by putting one\n * inner Observable after the other.</span>\n *\n * ![](concatAll.svg)\n *\n * Joins every Observable emitted by the source (a higher-order Observable), in\n * a serial fashion. It subscribes to each inner Observable only after the\n * previous inner Observable has completed, and merges all of their values into\n * the returned observable.\n *\n * __Warning:__ If the source Observable emits Observables quickly and\n * endlessly, and the inner Observables it emits generally complete slower than\n * the source emits, you can run into memory issues as the incoming Observables\n * collect in an unbounded buffer.\n *\n * Note: `concatAll` is equivalent to `mergeAll` with concurrency parameter set\n * to `1`.\n *\n * ## Example\n *\n * For each click event, tick every second from 0 to 3, with no concurrency\n *\n * ```ts\n * import { fromEvent, map, interval, take, concatAll } from 'rxjs';\n *\n * const clicks = fromEvent(document, 'click');\n * const higherOrder = clicks.pipe(\n *   map(() => interval(1000).pipe(take(4)))\n * );\n * const firstOrder = higherOrder.pipe(concatAll());\n * firstOrder.subscribe(x => console.log(x));\n *\n * // Results in the following:\n * // (results are not concurrent)\n * // For every click on the \"document\" it will emit values 0 to 3 spaced\n * // on a 1000ms interval\n * // one click = 1000ms-> 0 -1000ms-> 1 -1000ms-> 2 -1000ms-> 3\n * ```\n *\n * @see {@link combineLatestAll}\n * @see {@link concat}\n * @see {@link concatMap}\n * @see {@link concatMapTo}\n * @see {@link exhaustAll}\n * @see {@link mergeAll}\n * @see {@link switchAll}\n * @see {@link switchMap}\n * @see {@link zipAll}\n *\n * @return A function that returns an Observable emitting values from all the\n * inner Observables concatenated.\n */\nexport function concatAll<O extends ObservableInput<any>>(): OperatorFunction<O, ObservedValueOf<O>> {\n  return mergeAll(1);\n}\n", "import { Observable } from '../Observable';\nimport { ObservableInputTuple, SchedulerLike } from '../types';\nimport { concatAll } from '../operators/concatAll';\nimport { popScheduler } from '../util/args';\nimport { from } from './from';\n\nexport function concat<T extends readonly unknown[]>(...inputs: [...ObservableInputTuple<T>]): Observable<T[number]>;\nexport function concat<T extends readonly unknown[]>(\n  ...inputsAndScheduler: [...ObservableInputTuple<T>, SchedulerLike]\n): Observable<T[number]>;\n\n/**\n * Creates an output Observable which sequentially emits all values from the first given\n * Observable and then moves on to the next.\n *\n * <span class=\"informal\">Concatenates multiple Observables together by\n * sequentially emitting their values, one Observable after the other.</span>\n *\n * ![](concat.png)\n *\n * `concat` joins multiple Observables together, by subscribing to them one at a time and\n * merging their results into the output Observable. You can pass either an array of\n * Observables, or put them directly as arguments. Passing an empty array will result\n * in Observable that completes immediately.\n *\n * `concat` will subscribe to first input Observable and emit all its values, without\n * changing or affecting them in any way. When that Observable completes, it will\n * subscribe to then next Observable passed and, again, emit its values. This will be\n * repeated, until the operator runs out of Observables. When last input Observable completes,\n * `concat` will complete as well. At any given moment only one Observable passed to operator\n * emits values. If you would like to emit values from passed Observables concurrently, check out\n * {@link merge} instead, especially with optional `concurrent` parameter. As a matter of fact,\n * `concat` is an equivalent of `merge` operator with `concurrent` parameter set to `1`.\n *\n * Note that if some input Observable never completes, `concat` will also never complete\n * and Observables following the one that did not complete will never be subscribed. On the other\n * hand, if some Observable simply completes immediately after it is subscribed, it will be\n * invisible for `concat`, which will just move on to the next Observable.\n *\n * If any Observable in chain errors, instead of passing control to the next Observable,\n * `concat` will error immediately as well. Observables that would be subscribed after\n * the one that emitted error, never will.\n *\n * If you pass to `concat` the same Observable many times, its stream of values\n * will be \"replayed\" on every subscription, which means you can repeat given Observable\n * as many times as you like. If passing the same Observable to `concat` 1000 times becomes tedious,\n * you can always use {@link repeat}.\n *\n * ## Examples\n *\n * Concatenate a timer counting from 0 to 3 with a synchronous sequence from 1 to 10\n *\n * ```ts\n * import { interval, take, range, concat } from 'rxjs';\n *\n * const timer = interval(1000).pipe(take(4));\n * const sequence = range(1, 10);\n * const result = concat(timer, sequence);\n * result.subscribe(x => console.log(x));\n *\n * // results in:\n * // 0 -1000ms-> 1 -1000ms-> 2 -1000ms-> 3 -immediate-> 1 ... 10\n * ```\n *\n * Concatenate 3 Observables\n *\n * ```ts\n * import { interval, take, concat } from 'rxjs';\n *\n * const timer1 = interval(1000).pipe(take(10));\n * const timer2 = interval(2000).pipe(take(6));\n * const timer3 = interval(500).pipe(take(10));\n *\n * const result = concat(timer1, timer2, timer3);\n * result.subscribe(x => console.log(x));\n *\n * // results in the following:\n * // (Prints to console sequentially)\n * // -1000ms-> 0 -1000ms-> 1 -1000ms-> ... 9\n * // -2000ms-> 0 -2000ms-> 1 -2000ms-> ... 5\n * // -500ms-> 0 -500ms-> 1 -500ms-> ... 9\n * ```\n *\n * Concatenate the same Observable to repeat it\n *\n * ```ts\n * import { interval, take, concat } from 'rxjs';\n *\n * const timer = interval(1000).pipe(take(2));\n *\n * concat(timer, timer) // concatenating the same Observable!\n *   .subscribe({\n *     next: value => console.log(value),\n *     complete: () => console.log('...and it is done!')\n *   });\n *\n * // Logs:\n * // 0 after 1s\n * // 1 after 2s\n * // 0 after 3s\n * // 1 after 4s\n * // '...and it is done!' also after 4s\n * ```\n *\n * @see {@link concatAll}\n * @see {@link concatMap}\n * @see {@link concatMapTo}\n * @see {@link startWith}\n * @see {@link endWith}\n *\n * @param args Input Observables to concatenate.\n */\nexport function concat(...args: any[]): Observable<unknown> {\n  return concatAll()(from(args, popScheduler(args)));\n}\n", "import { Observable } from '../Observable';\nimport { ObservedValueOf, ObservableInput } from '../types';\nimport { innerFrom } from './innerFrom';\n\n/**\n * Creates an Observable that, on subscribe, calls an Observable factory to\n * make an Observable for each new Observer.\n *\n * <span class=\"informal\">Creates the Observable lazily, that is, only when it\n * is subscribed.\n * </span>\n *\n * ![](defer.png)\n *\n * `defer` allows you to create an Observable only when the Observer\n * subscribes. It waits until an Observer subscribes to it, calls the given\n * factory function to get an Observable -- where a factory function typically\n * generates a new Observable -- and subscribes the Observer to this Observable.\n * In case the factory function returns a falsy value, then EMPTY is used as\n * Observable instead. Last but not least, an exception during the factory\n * function call is transferred to the Observer by calling `error`.\n *\n * ## Example\n *\n * Subscribe to either an Observable of clicks or an Observable of interval, at random\n *\n * ```ts\n * import { defer, fromEvent, interval } from 'rxjs';\n *\n * const clicksOrInterval = defer(() => {\n *   return Math.random() > 0.5\n *     ? fromEvent(document, 'click')\n *     : interval(1000);\n * });\n * clicksOrInterval.subscribe(x => console.log(x));\n *\n * // Results in the following behavior:\n * // If the result of Math.random() is greater than 0.5 it will listen\n * // for clicks anywhere on the \"document\"; when document is clicked it\n * // will log a MouseEvent object to the console. If the result is less\n * // than 0.5 it will emit ascending numbers, one every second(1000ms).\n * ```\n *\n * @see {@link Observable}\n *\n * @param {function(): ObservableInput} observableFactory The Observable\n * factory function to invoke for each Observer that subscribes to the output\n * Observable. May also return a Promise, which will be converted on the fly\n * to an Observable.\n * @return {Observable} An Observable whose Observers' subscriptions trigger\n * an invocation of the given Observable factory function.\n */\nexport function defer<R extends ObservableInput<any>>(observableFactory: () => R): Observable<ObservedValueOf<R>> {\n  return new Observable<ObservedValueOf<R>>((subscriber) => {\n    innerFrom(observableFactory()).subscribe(subscriber);\n  });\n}\n", "import { innerFrom } from '../observable/innerFrom';\nimport { Observable } from '../Observable';\nimport { mergeMap } from '../operators/mergeMap';\nimport { isArrayLike } from '../util/isArrayLike';\nimport { isFunction } from '../util/isFunction';\nimport { mapOneOrManyArgs } from '../util/mapOneOrManyArgs';\n\n// These constants are used to create handler registry functions using array mapping below.\nconst nodeEventEmitterMethods = ['addListener', 'removeListener'] as const;\nconst eventTargetMethods = ['addEventListener', 'removeEventListener'] as const;\nconst jqueryMethods = ['on', 'off'] as const;\n\nexport interface NodeStyleEventEmitter {\n  addListener(eventName: string | symbol, handler: NodeEventHandler): this;\n  removeListener(eventName: string | symbol, handler: NodeEventHandler): this;\n}\n\nexport type NodeEventHandler = (...args: any[]) => void;\n\n// For APIs that implement `addListener` and `removeListener` methods that may\n// not use the same arguments or return EventEmitter values\n// such as React Native\nexport interface NodeCompatibleEventEmitter {\n  addListener(eventName: string, handler: NodeEventHandler): void | {};\n  removeListener(eventName: string, handler: NodeEventHandler): void | {};\n}\n\n// Use handler types like those in @types/jquery. See:\n// https://github.com/DefinitelyTyped/DefinitelyTyped/blob/847731ba1d7fa6db6b911c0e43aa0afe596e7723/types/jquery/misc.d.ts#L6395\nexport interface JQueryStyleEventEmitter<TContext, T> {\n  on(eventName: string, handler: (this: TContext, t: T, ...args: any[]) => any): void;\n  off(eventName: string, handler: (this: TContext, t: T, ...args: any[]) => any): void;\n}\n\nexport interface EventListenerObject<E> {\n  handleEvent(evt: E): void;\n}\n\nexport interface HasEventTargetAddRemove<E> {\n  addEventListener(\n    type: string,\n    listener: ((evt: E) => void) | EventListenerObject<E> | null,\n    options?: boolean | AddEventListenerOptions\n  ): void;\n  removeEventListener(\n    type: string,\n    listener: ((evt: E) => void) | EventListenerObject<E> | null,\n    options?: EventListenerOptions | boolean\n  ): void;\n}\n\nexport interface EventListenerOptions {\n  capture?: boolean;\n  passive?: boolean;\n  once?: boolean;\n}\n\nexport interface AddEventListenerOptions extends EventListenerOptions {\n  once?: boolean;\n  passive?: boolean;\n}\n\nexport function fromEvent<T>(target: HasEventTargetAddRemove<T> | ArrayLike<HasEventTargetAddRemove<T>>, eventName: string): Observable<T>;\nexport function fromEvent<T, R>(\n  target: HasEventTargetAddRemove<T> | ArrayLike<HasEventTargetAddRemove<T>>,\n  eventName: string,\n  resultSelector: (event: T) => R\n): Observable<R>;\nexport function fromEvent<T>(\n  target: HasEventTargetAddRemove<T> | ArrayLike<HasEventTargetAddRemove<T>>,\n  eventName: string,\n  options: EventListenerOptions\n): Observable<T>;\nexport function fromEvent<T, R>(\n  target: HasEventTargetAddRemove<T> | ArrayLike<HasEventTargetAddRemove<T>>,\n  eventName: string,\n  options: EventListenerOptions,\n  resultSelector: (event: T) => R\n): Observable<R>;\n\nexport function fromEvent(target: NodeStyleEventEmitter | ArrayLike<NodeStyleEventEmitter>, eventName: string): Observable<unknown>;\n/** @deprecated Do not specify explicit type parameters. Signatures with type parameters that cannot be inferred will be removed in v8. */\nexport function fromEvent<T>(target: NodeStyleEventEmitter | ArrayLike<NodeStyleEventEmitter>, eventName: string): Observable<T>;\nexport function fromEvent<R>(\n  target: NodeStyleEventEmitter | ArrayLike<NodeStyleEventEmitter>,\n  eventName: string,\n  resultSelector: (...args: any[]) => R\n): Observable<R>;\n\nexport function fromEvent(\n  target: NodeCompatibleEventEmitter | ArrayLike<NodeCompatibleEventEmitter>,\n  eventName: string\n): Observable<unknown>;\n/** @deprecated Do not specify explicit type parameters. Signatures with type parameters that cannot be inferred will be removed in v8. */\nexport function fromEvent<T>(target: NodeCompatibleEventEmitter | ArrayLike<NodeCompatibleEventEmitter>, eventName: string): Observable<T>;\nexport function fromEvent<R>(\n  target: NodeCompatibleEventEmitter | ArrayLike<NodeCompatibleEventEmitter>,\n  eventName: string,\n  resultSelector: (...args: any[]) => R\n): Observable<R>;\n\nexport function fromEvent<T>(\n  target: JQueryStyleEventEmitter<any, T> | ArrayLike<JQueryStyleEventEmitter<any, T>>,\n  eventName: string\n): Observable<T>;\nexport function fromEvent<T, R>(\n  target: JQueryStyleEventEmitter<any, T> | ArrayLike<JQueryStyleEventEmitter<any, T>>,\n  eventName: string,\n  resultSelector: (value: T, ...args: any[]) => R\n): Observable<R>;\n\n/**\n * Creates an Observable that emits events of a specific type coming from the\n * given event target.\n *\n * <span class=\"informal\">Creates an Observable from DOM events, or Node.js\n * EventEmitter events or others.</span>\n *\n * ![](fromEvent.png)\n *\n * `fromEvent` accepts as a first argument event target, which is an object with methods\n * for registering event handler functions. As a second argument it takes string that indicates\n * type of event we want to listen for. `fromEvent` supports selected types of event targets,\n * which are described in detail below. If your event target does not match any of the ones listed,\n * you should use {@link fromEventPattern}, which can be used on arbitrary APIs.\n * When it comes to APIs supported by `fromEvent`, their methods for adding and removing event\n * handler functions have different names, but they all accept a string describing event type\n * and function itself, which will be called whenever said event happens.\n *\n * Every time resulting Observable is subscribed, event handler function will be registered\n * to event target on given event type. When that event fires, value\n * passed as a first argument to registered function will be emitted by output Observable.\n * When Observable is unsubscribed, function will be unregistered from event target.\n *\n * Note that if event target calls registered function with more than one argument, second\n * and following arguments will not appear in resulting stream. In order to get access to them,\n * you can pass to `fromEvent` optional project function, which will be called with all arguments\n * passed to event handler. Output Observable will then emit value returned by project function,\n * instead of the usual value.\n *\n * Remember that event targets listed below are checked via duck typing. It means that\n * no matter what kind of object you have and no matter what environment you work in,\n * you can safely use `fromEvent` on that object if it exposes described methods (provided\n * of course they behave as was described above). So for example if Node.js library exposes\n * event target which has the same method names as DOM EventTarget, `fromEvent` is still\n * a good choice.\n *\n * If the API you use is more callback then event handler oriented (subscribed\n * callback function fires only once and thus there is no need to manually\n * unregister it), you should use {@link bindCallback} or {@link bindNodeCallback}\n * instead.\n *\n * `fromEvent` supports following types of event targets:\n *\n * **DOM EventTarget**\n *\n * This is an object with `addEventListener` and `removeEventListener` methods.\n *\n * In the browser, `addEventListener` accepts - apart from event type string and event\n * handler function arguments - optional third parameter, which is either an object or boolean,\n * both used for additional configuration how and when passed function will be called. When\n * `fromEvent` is used with event target of that type, you can provide this values\n * as third parameter as well.\n *\n * **Node.js EventEmitter**\n *\n * An object with `addListener` and `removeListener` methods.\n *\n * **JQuery-style event target**\n *\n * An object with `on` and `off` methods\n *\n * **DOM NodeList**\n *\n * List of DOM Nodes, returned for example by `document.querySelectorAll` or `Node.childNodes`.\n *\n * Although this collection is not event target in itself, `fromEvent` will iterate over all Nodes\n * it contains and install event handler function in every of them. When returned Observable\n * is unsubscribed, function will be removed from all Nodes.\n *\n * **DOM HtmlCollection**\n *\n * Just as in case of NodeList it is a collection of DOM nodes. Here as well event handler function is\n * installed and removed in each of elements.\n *\n *\n * ## Examples\n *\n * Emit clicks happening on the DOM document\n *\n * ```ts\n * import { fromEvent } from 'rxjs';\n *\n * const clicks = fromEvent(document, 'click');\n * clicks.subscribe(x => console.log(x));\n *\n * // Results in:\n * // MouseEvent object logged to console every time a click\n * // occurs on the document.\n * ```\n *\n * Use `addEventListener` with capture option\n *\n * ```ts\n * import { fromEvent } from 'rxjs';\n *\n * const div = document.createElement('div');\n * div.style.cssText = 'width: 200px; height: 200px; background: #09c;';\n * document.body.appendChild(div);\n *\n * // note optional configuration parameter which will be passed to addEventListener\n * const clicksInDocument = fromEvent(document, 'click', { capture: true });\n * const clicksInDiv = fromEvent(div, 'click');\n *\n * clicksInDocument.subscribe(() => console.log('document'));\n * clicksInDiv.subscribe(() => console.log('div'));\n *\n * // By default events bubble UP in DOM tree, so normally\n * // when we would click on div in document\n * // \"div\" would be logged first and then \"document\".\n * // Since we specified optional `capture` option, document\n * // will catch event when it goes DOWN DOM tree, so console\n * // will log \"document\" and then \"div\".\n * ```\n *\n * @see {@link bindCallback}\n * @see {@link bindNodeCallback}\n * @see {@link fromEventPattern}\n *\n * @param {FromEventTarget<T>} target The DOM EventTarget, Node.js\n * EventEmitter, JQuery-like event target, NodeList or HTMLCollection to attach the event handler to.\n * @param {string} eventName The event name of interest, being emitted by the\n * `target`.\n * @param {EventListenerOptions} [options] Options to pass through to addEventListener\n * @return {Observable<T>}\n */\nexport function fromEvent<T>(\n  target: any,\n  eventName: string,\n  options?: EventListenerOptions | ((...args: any[]) => T),\n  resultSelector?: (...args: any[]) => T\n): Observable<T> {\n  if (isFunction(options)) {\n    resultSelector = options;\n    options = undefined;\n  }\n  if (resultSelector) {\n    return fromEvent<T>(target, eventName, options as EventListenerOptions).pipe(mapOneOrManyArgs(resultSelector));\n  }\n\n  // Figure out our add and remove methods. In order to do this,\n  // we are going to analyze the target in a preferred order, if\n  // the target matches a given signature, we take the two \"add\" and \"remove\"\n  // method names and apply them to a map to create opposite versions of the\n  // same function. This is because they all operate in duplicate pairs,\n  // `addListener(name, handler)`, `removeListener(name, handler)`, for example.\n  // The call only differs by method name, as to whether or not you're adding or removing.\n  const [add, remove] =\n    // If it is an EventTarget, we need to use a slightly different method than the other two patterns.\n    isEventTarget(target)\n      ? eventTargetMethods.map((methodName) => (handler: any) => target[methodName](eventName, handler, options as EventListenerOptions))\n      : // In all other cases, the call pattern is identical with the exception of the method names.\n      isNodeStyleEventEmitter(target)\n      ? nodeEventEmitterMethods.map(toCommonHandlerRegistry(target, eventName))\n      : isJQueryStyleEventEmitter(target)\n      ? jqueryMethods.map(toCommonHandlerRegistry(target, eventName))\n      : [];\n\n  // If add is falsy, it's because we didn't match a pattern above.\n  // Check to see if it is an ArrayLike, because if it is, we want to\n  // try to apply fromEvent to all of it's items. We do this check last,\n  // because there are may be some types that are both ArrayLike *and* implement\n  // event registry points, and we'd rather delegate to that when possible.\n  if (!add) {\n    if (isArrayLike(target)) {\n      return mergeMap((subTarget: any) => fromEvent(subTarget, eventName, options as EventListenerOptions))(\n        innerFrom(target)\n      ) as Observable<T>;\n    }\n  }\n\n  // If add is falsy and we made it here, it's because we didn't\n  // match any valid target objects above.\n  if (!add) {\n    throw new TypeError('Invalid event target');\n  }\n\n  return new Observable<T>((subscriber) => {\n    // The handler we are going to register. Forwards the event object, by itself, or\n    // an array of arguments to the event handler, if there is more than one argument,\n    // to the consumer.\n    const handler = (...args: any[]) => subscriber.next(1 < args.length ? args : args[0]);\n    // Do the work of adding the handler to the target.\n    add(handler);\n    // When we finalize, we want to remove the handler and free up memory.\n    return () => remove!(handler);\n  });\n}\n\n/**\n * Used to create `add` and `remove` functions to register and unregister event handlers\n * from a target in the most common handler pattern, where there are only two arguments.\n * (e.g.  `on(name, fn)`, `off(name, fn)`, `addListener(name, fn)`, or `removeListener(name, fn)`)\n * @param target The target we're calling methods on\n * @param eventName The event name for the event we're creating register or unregister functions for\n */\nfunction toCommonHandlerRegistry(target: any, eventName: string) {\n  return (methodName: string) => (handler: any) => target[methodName](eventName, handler);\n}\n\n/**\n * Checks to see if the target implements the required node-style EventEmitter methods\n * for adding and removing event handlers.\n * @param target the object to check\n */\nfunction isNodeStyleEventEmitter(target: any): target is NodeStyleEventEmitter {\n  return isFunction(target.addListener) && isFunction(target.removeListener);\n}\n\n/**\n * Checks to see if the target implements the required jQuery-style EventEmitter methods\n * for adding and removing event handlers.\n * @param target the object to check\n */\nfunction isJQueryStyleEventEmitter(target: any): target is JQueryStyleEventEmitter<any, any> {\n  return isFunction(target.on) && isFunction(target.off);\n}\n\n/**\n * Checks to see if the target implements the required EventTarget methods\n * for adding and removing event handlers.\n * @param target the object to check\n */\nfunction isEventTarget(target: any): target is HasEventTargetAddRemove<any> {\n  return isFunction(target.addEventListener) && isFunction(target.removeEventListener);\n}\n", "import { Observable } from '../Observable';\nimport { isFunction } from '../util/isFunction';\nimport { NodeEventHandler } from './fromEvent';\nimport { mapOneOrManyArgs } from '../util/mapOneOrManyArgs';\n\n/* tslint:disable:max-line-length */\nexport function fromEventPattern<T>(\n  addHandler: (handler: NodeEventHandler) => any,\n  removeHandler?: (handler: NodeEventHandler, signal?: any) => void\n): Observable<T>;\nexport function fromEventPattern<T>(\n  addHandler: (handler: NodeEventHandler) => any,\n  removeHandler?: (handler: NodeEventHandler, signal?: any) => void,\n  resultSelector?: (...args: any[]) => T\n): Observable<T>;\n/* tslint:enable:max-line-length */\n\n/**\n * Creates an Observable from an arbitrary API for registering event handlers.\n *\n * <span class=\"informal\">When that method for adding event handler was something {@link fromEvent}\n * was not prepared for.</span>\n *\n * ![](fromEventPattern.png)\n *\n * `fromEventPattern` allows you to convert into an Observable any API that supports registering handler functions\n * for events. It is similar to {@link fromEvent}, but far\n * more flexible. In fact, all use cases of {@link fromEvent} could be easily handled by\n * `fromEventPattern` (although in slightly more verbose way).\n *\n * This operator accepts as a first argument an `addHandler` function, which will be injected with\n * handler parameter. That handler is actually an event handler function that you now can pass\n * to API expecting it. `addHandler` will be called whenever Observable\n * returned by the operator is subscribed, so registering handler in API will not\n * necessarily happen when `fromEventPattern` is called.\n *\n * After registration, every time an event that we listen to happens,\n * Observable returned by `fromEventPattern` will emit value that event handler\n * function was called with. Note that if event handler was called with more\n * than one argument, second and following arguments will not appear in the Observable.\n *\n * If API you are using allows to unregister event handlers as well, you can pass to `fromEventPattern`\n * another function - `removeHandler` - as a second parameter. It will be injected\n * with the same handler function as before, which now you can use to unregister\n * it from the API. `removeHandler` will be called when consumer of resulting Observable\n * unsubscribes from it.\n *\n * In some APIs unregistering is actually handled differently. Method registering an event handler\n * returns some kind of token, which is later used to identify which function should\n * be unregistered or it itself has method that unregisters event handler.\n * If that is the case with your API, make sure token returned\n * by registering method is returned by `addHandler`. Then it will be passed\n * as a second argument to `removeHandler`, where you will be able to use it.\n *\n * If you need access to all event handler parameters (not only the first one),\n * or you need to transform them in any way, you can call `fromEventPattern` with optional\n * third parameter - project function which will accept all arguments passed to\n * event handler when it is called. Whatever is returned from project function will appear on\n * resulting stream instead of usual event handlers first argument. This means\n * that default project can be thought of as function that takes its first parameter\n * and ignores the rest.\n *\n * ## Examples\n *\n * Emits clicks happening on the DOM document\n *\n * ```ts\n * import { fromEventPattern } from 'rxjs';\n *\n * function addClickHandler(handler) {\n *   document.addEventListener('click', handler);\n * }\n *\n * function removeClickHandler(handler) {\n *   document.removeEventListener('click', handler);\n * }\n *\n * const clicks = fromEventPattern(\n *   addClickHandler,\n *   removeClickHandler\n * );\n * clicks.subscribe(x => console.log(x));\n *\n * // Whenever you click anywhere in the browser, DOM MouseEvent\n * // object will be logged.\n * ```\n *\n * Use with API that returns cancellation token\n *\n * ```ts\n * import { fromEventPattern } from 'rxjs';\n *\n * const token = someAPI.registerEventHandler(function() {});\n * someAPI.unregisterEventHandler(token); // this APIs cancellation method accepts\n *                                        // not handler itself, but special token.\n *\n * const someAPIObservable = fromEventPattern(\n *   function(handler) { return someAPI.registerEventHandler(handler); }, // Note that we return the token here...\n *   function(handler, token) { someAPI.unregisterEventHandler(token); }  // ...to then use it here.\n * );\n * ```\n *\n * Use with project function\n *\n * ```ts\n * import { fromEventPattern } from 'rxjs';\n *\n * someAPI.registerEventHandler((eventType, eventMessage) => {\n *   console.log(eventType, eventMessage); // Logs 'EVENT_TYPE' 'EVENT_MESSAGE' to console.\n * });\n *\n * const someAPIObservable = fromEventPattern(\n *   handler => someAPI.registerEventHandler(handler),\n *   handler => someAPI.unregisterEventHandler(handler)\n *   (eventType, eventMessage) => eventType + ' --- ' + eventMessage // without that function only 'EVENT_TYPE'\n * );                                                                // would be emitted by the Observable\n *\n * someAPIObservable.subscribe(value => console.log(value));\n *\n * // Logs:\n * // 'EVENT_TYPE --- EVENT_MESSAGE'\n * ```\n *\n * @see {@link fromEvent}\n * @see {@link bindCallback}\n * @see {@link bindNodeCallback}\n *\n * @param {function(handler: Function): any} addHandler A function that takes\n * a `handler` function as argument and attaches it somehow to the actual\n * source of events.\n * @param {function(handler: Function, token?: any): void} [removeHandler] A function that\n * takes a `handler` function as an argument and removes it from the event source. If `addHandler`\n * returns some kind of token, `removeHandler` function will have it as a second parameter.\n * @param {function(...args: any): T} [project] A function to\n * transform results. It takes the arguments from the event handler and\n * should return a single value.\n * @return {Observable<T>} Observable which, when an event happens, emits first parameter\n * passed to registered event handler. Alternatively it emits whatever project function returns\n * at that moment.\n */\nexport function fromEventPattern<T>(\n  addHandler: (handler: NodeEventHandler) => any,\n  removeHandler?: (handler: NodeEventHandler, signal?: any) => void,\n  resultSelector?: (...args: any[]) => T\n): Observable<T | T[]> {\n  if (resultSelector) {\n    return fromEventPattern<T>(addHandler, removeHandler).pipe(mapOneOrManyArgs(resultSelector));\n  }\n\n  return new Observable<T | T[]>((subscriber) => {\n    const handler = (...e: T[]) => subscriber.next(e.length === 1 ? e[0] : e);\n    const retValue = addHandler(handler);\n    return isFunction(removeHandler) ? () => removeHandler(handler, retValue) : undefined;\n  });\n}\n", "import { Observable } from '../Observable';\nimport { SchedulerLike } from '../types';\nimport { async as asyncScheduler } from '../scheduler/async';\nimport { isScheduler } from '../util/isScheduler';\nimport { isValidDate } from '../util/isDate';\n\n/**\n * Creates an observable that will wait for a specified time period, or exact date, before\n * emitting the number 0.\n *\n * <span class=\"informal\">Used to emit a notification after a delay.</span>\n *\n * This observable is useful for creating delays in code, or racing against other values\n * for ad-hoc timeouts.\n *\n * The `delay` is specified by default in milliseconds, however providing a custom scheduler could\n * create a different behavior.\n *\n * ## Examples\n *\n * Wait 3 seconds and start another observable\n *\n * You might want to use `timer` to delay subscription to an\n * observable by a set amount of time. Here we use a timer with\n * {@link concatMapTo} or {@link concatMap} in order to wait\n * a few seconds and start a subscription to a source.\n *\n * ```ts\n * import { of, timer, concatMap } from 'rxjs';\n *\n * // This could be any observable\n * const source = of(1, 2, 3);\n *\n * timer(3000)\n *   .pipe(concatMap(() => source))\n *   .subscribe(console.log);\n * ```\n *\n * Take all values until the start of the next minute\n *\n * Using a `Date` as the trigger for the first emission, you can\n * do things like wait until midnight to fire an event, or in this case,\n * wait until a new minute starts (chosen so the example wouldn't take\n * too long to run) in order to stop watching a stream. Leveraging\n * {@link takeUntil}.\n *\n * ```ts\n * import { interval, takeUntil, timer } from 'rxjs';\n *\n * // Build a Date object that marks the\n * // next minute.\n * const currentDate = new Date();\n * const startOfNextMinute = new Date(\n *   currentDate.getFullYear(),\n *   currentDate.getMonth(),\n *   currentDate.getDate(),\n *   currentDate.getHours(),\n *   currentDate.getMinutes() + 1\n * );\n *\n * // This could be any observable stream\n * const source = interval(1000);\n *\n * const result = source.pipe(\n *   takeUntil(timer(startOfNextMinute))\n * );\n *\n * result.subscribe(console.log);\n * ```\n *\n * ### Known Limitations\n *\n * - The {@link asyncScheduler} uses `setTimeout` which has limitations for how far in the future it can be scheduled.\n *\n * - If a `scheduler` is provided that returns a timestamp other than an epoch from `now()`, and\n * a `Date` object is passed to the `dueTime` argument, the calculation for when the first emission\n * should occur will be incorrect. In this case, it would be best to do your own calculations\n * ahead of time, and pass a `number` in as the `dueTime`.\n *\n * @param due If a `number`, the amount of time in milliseconds to wait before emitting.\n * If a `Date`, the exact time at which to emit.\n * @param scheduler The scheduler to use to schedule the delay. Defaults to {@link asyncScheduler}.\n */\nexport function timer(due: number | Date, scheduler?: SchedulerLike): Observable<0>;\n\n/**\n * Creates an observable that starts an interval after a specified delay, emitting incrementing numbers -- starting at `0` --\n * on each interval after words.\n *\n * The `delay` and `intervalDuration` are specified by default in milliseconds, however providing a custom scheduler could\n * create a different behavior.\n *\n * ## Example\n *\n * ### Start an interval that starts right away\n *\n * Since {@link interval} waits for the passed delay before starting,\n * sometimes that's not ideal. You may want to start an interval immediately.\n * `timer` works well for this. Here we have both side-by-side so you can\n * see them in comparison.\n *\n * Note that this observable will never complete.\n *\n * ```ts\n * import { timer, interval } from 'rxjs';\n *\n * timer(0, 1000).subscribe(n => console.log('timer', n));\n * interval(1000).subscribe(n => console.log('interval', n));\n * ```\n *\n * ### Known Limitations\n *\n * - The {@link asyncScheduler} uses `setTimeout` which has limitations for how far in the future it can be scheduled.\n *\n * - If a `scheduler` is provided that returns a timestamp other than an epoch from `now()`, and\n * a `Date` object is passed to the `dueTime` argument, the calculation for when the first emission\n * should occur will be incorrect. In this case, it would be best to do your own calculations\n * ahead of time, and pass a `number` in as the `startDue`.\n * @param startDue If a `number`, is the time to wait before starting the interval.\n * If a `Date`, is the exact time at which to start the interval.\n * @param intervalDuration The delay between each value emitted in the interval. Passing a\n * negative number here will result in immediate completion after the first value is emitted, as though\n * no `intervalDuration` was passed at all.\n * @param scheduler The scheduler to use to schedule the delay. Defaults to {@link asyncScheduler}.\n */\nexport function timer(startDue: number | Date, intervalDuration: number, scheduler?: SchedulerLike): Observable<number>;\n\n/**\n * @deprecated The signature allowing `undefined` to be passed for `intervalDuration` will be removed in v8. Use the `timer(dueTime, scheduler?)` signature instead.\n */\nexport function timer(dueTime: number | Date, unused: undefined, scheduler?: SchedulerLike): Observable<0>;\n\nexport function timer(\n  dueTime: number | Date = 0,\n  intervalOrScheduler?: number | SchedulerLike,\n  scheduler: SchedulerLike = asyncScheduler\n): Observable<number> {\n  // Since negative intervalDuration is treated as though no\n  // interval was specified at all, we start with a negative number.\n  let intervalDuration = -1;\n\n  if (intervalOrScheduler != null) {\n    // If we have a second argument, and it's a scheduler,\n    // override the scheduler we had defaulted. Otherwise,\n    // it must be an interval.\n    if (isScheduler(intervalOrScheduler)) {\n      scheduler = intervalOrScheduler;\n    } else {\n      // Note that this *could* be negative, in which case\n      // it's like not passing an intervalDuration at all.\n      intervalDuration = intervalOrScheduler;\n    }\n  }\n\n  return new Observable((subscriber) => {\n    // If a valid date is passed, calculate how long to wait before\n    // executing the first value... otherwise, if it's a number just schedule\n    // that many milliseconds (or scheduler-specified unit size) in the future.\n    let due = isValidDate(dueTime) ? +dueTime - scheduler!.now() : dueTime;\n\n    if (due < 0) {\n      // Ensure we don't schedule in the future.\n      due = 0;\n    }\n\n    // The incrementing value we emit.\n    let n = 0;\n\n    // Start the timer.\n    return scheduler.schedule(function () {\n      if (!subscriber.closed) {\n        // Emit the next value and increment.\n        subscriber.next(n++);\n\n        if (0 <= intervalDuration) {\n          // If we have a interval after the initial timer,\n          // reschedule with the period.\n          this.schedule(undefined, intervalDuration);\n        } else {\n          // We didn't have an interval. So just complete.\n          subscriber.complete();\n        }\n      }\n    }, due);\n  });\n}\n", "import { Observable } from '../Observable';\nimport { ObservableInput, ObservableInputTuple, SchedulerLike } from '../types';\nimport { mergeAll } from '../operators/mergeAll';\nimport { innerFrom } from './innerFrom';\nimport { EMPTY } from './empty';\nimport { popNumber, popScheduler } from '../util/args';\nimport { from } from './from';\n\nexport function merge<A extends readonly unknown[]>(...sources: [...ObservableInputTuple<A>]): Observable<A[number]>;\nexport function merge<A extends readonly unknown[]>(...sourcesAndConcurrency: [...ObservableInputTuple<A>, number?]): Observable<A[number]>;\n/** @deprecated The `scheduler` parameter will be removed in v8. Use `scheduled` and `mergeAll`. Details: https://rxjs.dev/deprecations/scheduler-argument */\nexport function merge<A extends readonly unknown[]>(\n  ...sourcesAndScheduler: [...ObservableInputTuple<A>, SchedulerLike?]\n): Observable<A[number]>;\n/** @deprecated The `scheduler` parameter will be removed in v8. Use `scheduled` and `mergeAll`. Details: https://rxjs.dev/deprecations/scheduler-argument */\nexport function merge<A extends readonly unknown[]>(\n  ...sourcesAndConcurrencyAndScheduler: [...ObservableInputTuple<A>, number?, SchedulerLike?]\n): Observable<A[number]>;\n\n/**\n * Creates an output Observable which concurrently emits all values from every\n * given input Observable.\n *\n * <span class=\"informal\">Flattens multiple Observables together by blending\n * their values into one Observable.</span>\n *\n * ![](merge.png)\n *\n * `merge` subscribes to each given input Observable (as arguments), and simply\n * forwards (without doing any transformation) all the values from all the input\n * Observables to the output Observable. The output Observable only completes\n * once all input Observables have completed. Any error delivered by an input\n * Observable will be immediately emitted on the output Observable.\n *\n * ## Examples\n *\n * Merge together two Observables: 1s interval and clicks\n *\n * ```ts\n * import { merge, fromEvent, interval } from 'rxjs';\n *\n * const clicks = fromEvent(document, 'click');\n * const timer = interval(1000);\n * const clicksOrTimer = merge(clicks, timer);\n * clicksOrTimer.subscribe(x => console.log(x));\n *\n * // Results in the following:\n * // timer will emit ascending values, one every second(1000ms) to console\n * // clicks logs MouseEvents to console every time the \"document\" is clicked\n * // Since the two streams are merged you see these happening\n * // as they occur.\n * ```\n *\n * Merge together 3 Observables, but run only 2 concurrently\n *\n * ```ts\n * import { interval, take, merge } from 'rxjs';\n *\n * const timer1 = interval(1000).pipe(take(10));\n * const timer2 = interval(2000).pipe(take(6));\n * const timer3 = interval(500).pipe(take(10));\n *\n * const concurrent = 2; // the argument\n * const merged = merge(timer1, timer2, timer3, concurrent);\n * merged.subscribe(x => console.log(x));\n *\n * // Results in the following:\n * // - First timer1 and timer2 will run concurrently\n * // - timer1 will emit a value every 1000ms for 10 iterations\n * // - timer2 will emit a value every 2000ms for 6 iterations\n * // - after timer1 hits its max iteration, timer2 will\n * //   continue, and timer3 will start to run concurrently with timer2\n * // - when timer2 hits its max iteration it terminates, and\n * //   timer3 will continue to emit a value every 500ms until it is complete\n * ```\n *\n * @see {@link mergeAll}\n * @see {@link mergeMap}\n * @see {@link mergeMapTo}\n * @see {@link mergeScan}\n *\n * @param {...ObservableInput} observables Input Observables to merge together.\n * @param {number} [concurrent=Infinity] Maximum number of input\n * Observables being subscribed to concurrently.\n * @param {SchedulerLike} [scheduler=null] The {@link SchedulerLike} to use for managing\n * concurrency of input Observables.\n * @return {Observable} an Observable that emits items that are the result of\n * every input Observable.\n */\nexport function merge(...args: (ObservableInput<unknown> | number | SchedulerLike)[]): Observable<unknown> {\n  const scheduler = popScheduler(args);\n  const concurrent = popNumber(args, Infinity);\n  const sources = args as ObservableInput<unknown>[];\n  return !sources.length\n    ? // No source provided\n      EMPTY\n    : sources.length === 1\n    ? // One source? Just return it.\n      innerFrom(sources[0])\n    : // Merge all sources\n      mergeAll(concurrent)(from(sources, scheduler));\n}\n", "import { Observable } from '../Observable';\nimport { noop } from '../util/noop';\n\n/**\n * An Observable that emits no items to the Observer and never completes.\n *\n * ![](never.png)\n *\n * A simple Observable that emits neither values nor errors nor the completion\n * notification. It can be used for testing purposes or for composing with other\n * Observables. Please note that by never emitting a complete notification, this\n * Observable keeps the subscription from being disposed automatically.\n * Subscriptions need to be manually disposed.\n *\n * ##  Example\n *\n * Emit the number 7, then never emit anything else (not even complete)\n *\n * ```ts\n * import { NEVER, startWith } from 'rxjs';\n *\n * const info = () => console.log('Will not be called');\n *\n * const result = NEVER.pipe(startWith(7));\n * result.subscribe({\n *   next: x => console.log(x),\n *   error: info,\n *   complete: info\n * });\n * ```\n *\n * @see {@link Observable}\n * @see {@link EMPTY}\n * @see {@link of}\n * @see {@link throwError}\n */\nexport const NEVER = new Observable<never>(noop);\n\n/**\n * @deprecated Replaced with the {@link NEVER} constant. Will be removed in v8.\n */\nexport function never() {\n  return NEVER;\n}\n", "const { isArray } = Array;\n\n/**\n * Used in operators and functions that accept either a list of arguments, or an array of arguments\n * as a single argument.\n */\nexport function argsOrArgArray<T>(args: (T | T[])[]): T[] {\n  return args.length === 1 && isArray(args[0]) ? args[0] : (args as T[]);\n}\n", "import { OperatorFunction, MonoTypeOperatorFunction, TruthyTypesOf } from '../types';\nimport { operate } from '../util/lift';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\n\n/** @deprecated Use a closure instead of a `thisArg`. Signatures accepting a `thisArg` will be removed in v8. */\nexport function filter<T, S extends T, A>(predicate: (this: A, value: T, index: number) => value is S, thisArg: A): OperatorFunction<T, S>;\nexport function filter<T, S extends T>(predicate: (value: T, index: number) => value is S): OperatorFunction<T, S>;\nexport function filter<T>(predicate: BooleanConstructor): OperatorFunction<T, TruthyTypesOf<T>>;\n/** @deprecated Use a closure instead of a `thisArg`. Signatures accepting a `thisArg` will be removed in v8. */\nexport function filter<T, A>(predicate: (this: A, value: T, index: number) => boolean, thisArg: A): MonoTypeOperatorFunction<T>;\nexport function filter<T>(predicate: (value: T, index: number) => boolean): MonoTypeOperatorFunction<T>;\n\n/**\n * Filter items emitted by the source Observable by only emitting those that\n * satisfy a specified predicate.\n *\n * <span class=\"informal\">Like\n * [Array.prototype.filter()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/filter),\n * it only emits a value from the source if it passes a criterion function.</span>\n *\n * ![](filter.png)\n *\n * Similar to the well-known `Array.prototype.filter` method, this operator\n * takes values from the source Observable, passes them through a `predicate`\n * function and only emits those values that yielded `true`.\n *\n * ## Example\n *\n * Emit only click events whose target was a DIV element\n *\n * ```ts\n * import { fromEvent, filter } from 'rxjs';\n *\n * const div = document.createElement('div');\n * div.style.cssText = 'width: 200px; height: 200px; background: #09c;';\n * document.body.appendChild(div);\n *\n * const clicks = fromEvent(document, 'click');\n * const clicksOnDivs = clicks.pipe(filter(ev => (<HTMLElement>ev.target).tagName === 'DIV'));\n * clicksOnDivs.subscribe(x => console.log(x));\n * ```\n *\n * @see {@link distinct}\n * @see {@link distinctUntilChanged}\n * @see {@link distinctUntilKeyChanged}\n * @see {@link ignoreElements}\n * @see {@link partition}\n * @see {@link skip}\n *\n * @param predicate A function that\n * evaluates each value emitted by the source Observable. If it returns `true`,\n * the value is emitted, if `false` the value is not passed to the output\n * Observable. The `index` parameter is the number `i` for the i-th source\n * emission that has happened since the subscription, starting from the number\n * `0`.\n * @param thisArg An optional argument to determine the value of `this`\n * in the `predicate` function.\n * @return A function that returns an Observable that emits items from the\n * source Observable that satisfy the specified `predicate`.\n */\nexport function filter<T>(predicate: (value: T, index: number) => boolean, thisArg?: any): MonoTypeOperatorFunction<T> {\n  return operate((source, subscriber) => {\n    // An index passed to our predicate function on each call.\n    let index = 0;\n\n    // Subscribe to the source, all errors and completions are\n    // forwarded to the consumer.\n    source.subscribe(\n      // Call the predicate with the appropriate `this` context,\n      // if the predicate returns `true`, then send the value\n      // to the consumer.\n      createOperatorSubscriber(subscriber, (value) => predicate.call(thisArg, value, index++) && subscriber.next(value))\n    );\n  });\n}\n", "import { Observable } from '../Observable';\nimport { ObservableInputTuple } from '../types';\nimport { innerFrom } from './innerFrom';\nimport { argsOrArgArray } from '../util/argsOrArgArray';\nimport { EMPTY } from './empty';\nimport { createOperatorSubscriber } from '../operators/OperatorSubscriber';\nimport { popResultSelector } from '../util/args';\n\nexport function zip<A extends readonly unknown[]>(sources: [...ObservableInputTuple<A>]): Observable<A>;\nexport function zip<A extends readonly unknown[], R>(\n  sources: [...ObservableInputTuple<A>],\n  resultSelector: (...values: A) => R\n): Observable<R>;\nexport function zip<A extends readonly unknown[]>(...sources: [...ObservableInputTuple<A>]): Observable<A>;\nexport function zip<A extends readonly unknown[], R>(\n  ...sourcesAndResultSelector: [...ObservableInputTuple<A>, (...values: A) => R]\n): Observable<R>;\n\n/**\n * Combines multiple Observables to create an Observable whose values are calculated from the values, in order, of each\n * of its input Observables.\n *\n * If the last parameter is a function, this function is used to compute the created value from the input values.\n * Otherwise, an array of the input values is returned.\n *\n * ## Example\n *\n * Combine age and name from different sources\n *\n * ```ts\n * import { of, zip, map } from 'rxjs';\n *\n * const age$ = of(27, 25, 29);\n * const name$ = of('Foo', 'Bar', 'Beer');\n * const isDev$ = of(true, true, false);\n *\n * zip(age$, name$, isDev$).pipe(\n *   map(([age, name, isDev]) => ({ age, name, isDev }))\n * )\n * .subscribe(x => console.log(x));\n *\n * // Outputs\n * // { age: 27, name: 'Foo', isDev: true }\n * // { age: 25, name: 'Bar', isDev: true }\n * // { age: 29, name: 'Beer', isDev: false }\n * ```\n *\n * @param sources\n * @return {Observable<R>}\n */\nexport function zip(...args: unknown[]): Observable<unknown> {\n  const resultSelector = popResultSelector(args);\n\n  const sources = argsOrArgArray(args) as Observable<unknown>[];\n\n  return sources.length\n    ? new Observable<unknown[]>((subscriber) => {\n        // A collection of buffers of values from each source.\n        // Keyed by the same index with which the sources were passed in.\n        let buffers: unknown[][] = sources.map(() => []);\n\n        // An array of flags of whether or not the sources have completed.\n        // This is used to check to see if we should complete the result.\n        // Keyed by the same index with which the sources were passed in.\n        let completed = sources.map(() => false);\n\n        // When everything is done, release the arrays above.\n        subscriber.add(() => {\n          buffers = completed = null!;\n        });\n\n        // Loop over our sources and subscribe to each one. The index `i` is\n        // especially important here, because we use it in closures below to\n        // access the related buffers and completion properties\n        for (let sourceIndex = 0; !subscriber.closed && sourceIndex < sources.length; sourceIndex++) {\n          innerFrom(sources[sourceIndex]).subscribe(\n            createOperatorSubscriber(\n              subscriber,\n              (value) => {\n                buffers[sourceIndex].push(value);\n                // if every buffer has at least one value in it, then we\n                // can shift out the oldest value from each buffer and emit\n                // them as an array.\n                if (buffers.every((buffer) => buffer.length)) {\n                  const result: any = buffers.map((buffer) => buffer.shift()!);\n                  // Emit the array. If theres' a result selector, use that.\n                  subscriber.next(resultSelector ? resultSelector(...result) : result);\n                  // If any one of the sources is both complete and has an empty buffer\n                  // then we complete the result. This is because we cannot possibly have\n                  // any more values to zip together.\n                  if (buffers.some((buffer, i) => !buffer.length && completed[i])) {\n                    subscriber.complete();\n                  }\n                }\n              },\n              () => {\n                // This source completed. Mark it as complete so we can check it later\n                // if we have to.\n                completed[sourceIndex] = true;\n                // But, if this complete source has nothing in its buffer, then we\n                // can complete the result, because we can't possibly have any more\n                // values from this to zip together with the other values.\n                !buffers[sourceIndex].length && subscriber.complete();\n              }\n            )\n          );\n        }\n\n        // When everything is done, release the arrays above.\n        return () => {\n          buffers = completed = null!;\n        };\n      })\n    : EMPTY;\n}\n", "import { Subscriber } from '../Subscriber';\nimport { MonoTypeOperatorFunction, ObservableInput } from '../types';\n\nimport { operate } from '../util/lift';\nimport { innerFrom } from '../observable/innerFrom';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\n\n/**\n * Ignores source values for a duration determined by another Observable, then\n * emits the most recent value from the source Observable, then repeats this\n * process.\n *\n * <span class=\"informal\">It's like {@link auditTime}, but the silencing\n * duration is determined by a second Observable.</span>\n *\n * ![](audit.svg)\n *\n * `audit` is similar to `throttle`, but emits the last value from the silenced\n * time window, instead of the first value. `audit` emits the most recent value\n * from the source Observable on the output Observable as soon as its internal\n * timer becomes disabled, and ignores source values while the timer is enabled.\n * Initially, the timer is disabled. As soon as the first source value arrives,\n * the timer is enabled by calling the `durationSelector` function with the\n * source value, which returns the \"duration\" Observable. When the duration\n * Observable emits a value, the timer is disabled, then the most\n * recent source value is emitted on the output Observable, and this process\n * repeats for the next source value.\n *\n * ## Example\n *\n * Emit clicks at a rate of at most one click per second\n *\n * ```ts\n * import { fromEvent, audit, interval } from 'rxjs';\n *\n * const clicks = fromEvent(document, 'click');\n * const result = clicks.pipe(audit(ev => interval(1000)));\n * result.subscribe(x => console.log(x));\n * ```\n *\n * @see {@link auditTime}\n * @see {@link debounce}\n * @see {@link delayWhen}\n * @see {@link sample}\n * @see {@link throttle}\n *\n * @param durationSelector A function\n * that receives a value from the source Observable, for computing the silencing\n * duration, returned as an Observable or a Promise.\n * @return A function that returns an Observable that performs rate-limiting of\n * emissions from the source Observable.\n */\nexport function audit<T>(durationSelector: (value: T) => ObservableInput<any>): MonoTypeOperatorFunction<T> {\n  return operate((source, subscriber) => {\n    let hasValue = false;\n    let lastValue: T | null = null;\n    let durationSubscriber: Subscriber<any> | null = null;\n    let isComplete = false;\n\n    const endDuration = () => {\n      durationSubscriber?.unsubscribe();\n      durationSubscriber = null;\n      if (hasValue) {\n        hasValue = false;\n        const value = lastValue!;\n        lastValue = null;\n        subscriber.next(value);\n      }\n      isComplete && subscriber.complete();\n    };\n\n    const cleanupDuration = () => {\n      durationSubscriber = null;\n      isComplete && subscriber.complete();\n    };\n\n    source.subscribe(\n      createOperatorSubscriber(\n        subscriber,\n        (value) => {\n          hasValue = true;\n          lastValue = value;\n          if (!durationSubscriber) {\n            innerFrom(durationSelector(value)).subscribe(\n              (durationSubscriber = createOperatorSubscriber(subscriber, endDuration, cleanupDuration))\n            );\n          }\n        },\n        () => {\n          isComplete = true;\n          (!hasValue || !durationSubscriber || durationSubscriber.closed) && subscriber.complete();\n        }\n      )\n    );\n  });\n}\n", "import { asyncScheduler } from '../scheduler/async';\nimport { audit } from './audit';\nimport { timer } from '../observable/timer';\nimport { MonoTypeOperatorFunction, SchedulerLike } from '../types';\n\n/**\n * Ignores source values for `duration` milliseconds, then emits the most recent\n * value from the source Observable, then repeats this process.\n *\n * <span class=\"informal\">When it sees a source value, it ignores that plus\n * the next ones for `duration` milliseconds, and then it emits the most recent\n * value from the source.</span>\n *\n * ![](auditTime.png)\n *\n * `auditTime` is similar to `throttleTime`, but emits the last value from the\n * silenced time window, instead of the first value. `auditTime` emits the most\n * recent value from the source Observable on the output Observable as soon as\n * its internal timer becomes disabled, and ignores source values while the\n * timer is enabled. Initially, the timer is disabled. As soon as the first\n * source value arrives, the timer is enabled. After `duration` milliseconds (or\n * the time unit determined internally by the optional `scheduler`) has passed,\n * the timer is disabled, then the most recent source value is emitted on the\n * output Observable, and this process repeats for the next source value.\n * Optionally takes a {@link SchedulerLike} for managing timers.\n *\n * ## Example\n *\n * Emit clicks at a rate of at most one click per second\n *\n * ```ts\n * import { fromEvent, auditTime } from 'rxjs';\n *\n * const clicks = fromEvent(document, 'click');\n * const result = clicks.pipe(auditTime(1000));\n * result.subscribe(x => console.log(x));\n * ```\n *\n * @see {@link audit}\n * @see {@link debounceTime}\n * @see {@link delay}\n * @see {@link sampleTime}\n * @see {@link throttleTime}\n *\n * @param {number} duration Time to wait before emitting the most recent source\n * value, measured in milliseconds or the time unit determined internally\n * by the optional `scheduler`.\n * @param {SchedulerLike} [scheduler=async] The {@link SchedulerLike} to use for\n * managing the timers that handle the rate-limiting behavior.\n * @return A function that returns an Observable that performs rate-limiting of\n * emissions from the source Observable.\n */\nexport function auditTime<T>(duration: number, scheduler: SchedulerLike = asyncScheduler): MonoTypeOperatorFunction<T> {\n  return audit(() => timer(duration, scheduler));\n}\n", "import { OperatorFunction } from '../types';\nimport { operate } from '../util/lift';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\nimport { arrRemove } from '../util/arrRemove';\n\n/**\n * Buffers the source Observable values until the size hits the maximum\n * `bufferSize` given.\n *\n * <span class=\"informal\">Collects values from the past as an array, and emits\n * that array only when its size reaches `bufferSize`.</span>\n *\n * ![](bufferCount.png)\n *\n * Buffers a number of values from the source Observable by `bufferSize` then\n * emits the buffer and clears it, and starts a new buffer each\n * `startBufferEvery` values. If `startBufferEvery` is not provided or is\n * `null`, then new buffers are started immediately at the start of the source\n * and when each buffer closes and is emitted.\n *\n * ## Examples\n *\n * Emit the last two click events as an array\n *\n * ```ts\n * import { fromEvent, bufferCount } from 'rxjs';\n *\n * const clicks = fromEvent(document, 'click');\n * const buffered = clicks.pipe(bufferCount(2));\n * buffered.subscribe(x => console.log(x));\n * ```\n *\n * On every click, emit the last two click events as an array\n *\n * ```ts\n * import { fromEvent, bufferCount } from 'rxjs';\n *\n * const clicks = fromEvent(document, 'click');\n * const buffered = clicks.pipe(bufferCount(2, 1));\n * buffered.subscribe(x => console.log(x));\n * ```\n *\n * @see {@link buffer}\n * @see {@link bufferTime}\n * @see {@link bufferToggle}\n * @see {@link bufferWhen}\n * @see {@link pairwise}\n * @see {@link windowCount}\n *\n * @param {number} bufferSize The maximum size of the buffer emitted.\n * @param {number} [startBufferEvery] Interval at which to start a new buffer.\n * For example if `startBufferEvery` is `2`, then a new buffer will be started\n * on every other value from the source. A new buffer is started at the\n * beginning of the source by default.\n * @return A function that returns an Observable of arrays of buffered values.\n */\nexport function bufferCount<T>(bufferSize: number, startBufferEvery: number | null = null): OperatorFunction<T, T[]> {\n  // If no `startBufferEvery` value was supplied, then we're\n  // opening and closing on the bufferSize itself.\n  startBufferEvery = startBufferEvery ?? bufferSize;\n\n  return operate((source, subscriber) => {\n    let buffers: T[][] = [];\n    let count = 0;\n\n    source.subscribe(\n      createOperatorSubscriber(\n        subscriber,\n        (value) => {\n          let toEmit: T[][] | null = null;\n\n          // Check to see if we need to start a buffer.\n          // This will start one at the first value, and then\n          // a new one every N after that.\n          if (count++ % startBufferEvery! === 0) {\n            buffers.push([]);\n          }\n\n          // Push our value into our active buffers.\n          for (const buffer of buffers) {\n            buffer.push(value);\n            // Check to see if we're over the bufferSize\n            // if we are, record it so we can emit it later.\n            // If we emitted it now and removed it, it would\n            // mutate the `buffers` array while we're looping\n            // over it.\n            if (bufferSize <= buffer.length) {\n              toEmit = toEmit ?? [];\n              toEmit.push(buffer);\n            }\n          }\n\n          if (toEmit) {\n            // We have found some buffers that are over the\n            // `bufferSize`. Emit them, and remove them from our\n            // buffers list.\n            for (const buffer of toEmit) {\n              arrRemove(buffers, buffer);\n              subscriber.next(buffer);\n            }\n          }\n        },\n        () => {\n          // When the source completes, emit all of our\n          // active buffers.\n          for (const buffer of buffers) {\n            subscriber.next(buffer);\n          }\n          subscriber.complete();\n        },\n        // Pass all errors through to consumer.\n        undefined,\n        () => {\n          // Clean up our memory when we finalize\n          buffers = null!;\n        }\n      )\n    );\n  });\n}\n", "import { Observable } from '../Observable';\n\nimport { ObservableInput, OperatorFunction, ObservedValueOf } from '../types';\nimport { Subscription } from '../Subscription';\nimport { innerFrom } from '../observable/innerFrom';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\nimport { operate } from '../util/lift';\n\n/* tslint:disable:max-line-length */\nexport function catchError<T, O extends ObservableInput<any>>(\n  selector: (err: any, caught: Observable<T>) => O\n): OperatorFunction<T, T | ObservedValueOf<O>>;\n/* tslint:enable:max-line-length */\n\n/**\n * Catches errors on the observable to be handled by returning a new observable or throwing an error.\n *\n * <span class=\"informal\">\n * It only listens to the error channel and ignores notifications.\n * Handles errors from the source observable, and maps them to a new observable.\n * The error may also be rethrown, or a new error can be thrown to emit an error from the result.\n * </span>\n *\n * ![](catch.png)\n *\n * This operator handles errors, but forwards along all other events to the resulting observable.\n * If the source observable terminates with an error, it will map that error to a new observable,\n * subscribe to it, and forward all of its events to the resulting observable.\n *\n * ## Examples\n *\n * Continue with a different Observable when there's an error\n *\n * ```ts\n * import { of, map, catchError } from 'rxjs';\n *\n * of(1, 2, 3, 4, 5)\n *   .pipe(\n *     map(n => {\n *       if (n === 4) {\n *         throw 'four!';\n *       }\n *       return n;\n *     }),\n *     catchError(err => of('I', 'II', 'III', 'IV', 'V'))\n *   )\n *   .subscribe(x => console.log(x));\n *   // 1, 2, 3, I, II, III, IV, V\n * ```\n *\n * Retry the caught source Observable again in case of error, similar to `retry()` operator\n *\n * ```ts\n * import { of, map, catchError, take } from 'rxjs';\n *\n * of(1, 2, 3, 4, 5)\n *   .pipe(\n *     map(n => {\n *       if (n === 4) {\n *         throw 'four!';\n *       }\n *       return n;\n *     }),\n *     catchError((err, caught) => caught),\n *     take(30)\n *   )\n *   .subscribe(x => console.log(x));\n *   // 1, 2, 3, 1, 2, 3, ...\n * ```\n *\n * Throw a new error when the source Observable throws an error\n *\n * ```ts\n * import { of, map, catchError } from 'rxjs';\n *\n * of(1, 2, 3, 4, 5)\n *   .pipe(\n *     map(n => {\n *       if (n === 4) {\n *         throw 'four!';\n *       }\n *       return n;\n *     }),\n *     catchError(err => {\n *       throw 'error in source. Details: ' + err;\n *     })\n *   )\n *   .subscribe({\n *     next: x => console.log(x),\n *     error: err => console.log(err)\n *   });\n *   // 1, 2, 3, error in source. Details: four!\n * ```\n *\n * @see {@link onErrorResumeNext}\n * @see {@link repeat}\n * @see {@link repeatWhen}\n * @see {@link retry }\n * @see {@link retryWhen}\n *\n * @param {function} selector a function that takes as arguments `err`, which is the error, and `caught`, which\n * is the source observable, in case you'd like to \"retry\" that observable by returning it again. Whatever observable\n * is returned by the `selector` will be used to continue the observable chain.\n * @return A function that returns an Observable that originates from either\n * the source or the Observable returned by the `selector` function.\n */\nexport function catchError<T, O extends ObservableInput<any>>(\n  selector: (err: any, caught: Observable<T>) => O\n): OperatorFunction<T, T | ObservedValueOf<O>> {\n  return operate((source, subscriber) => {\n    let innerSub: Subscription | null = null;\n    let syncUnsub = false;\n    let handledResult: Observable<ObservedValueOf<O>>;\n\n    innerSub = source.subscribe(\n      createOperatorSubscriber(subscriber, undefined, undefined, (err) => {\n        handledResult = innerFrom(selector(err, catchError(selector)(source)));\n        if (innerSub) {\n          innerSub.unsubscribe();\n          innerSub = null;\n          handledResult.subscribe(subscriber);\n        } else {\n          // We don't have an innerSub yet, that means the error was synchronous\n          // because the subscribe call hasn't returned yet.\n          syncUnsub = true;\n        }\n      })\n    );\n\n    if (syncUnsub) {\n      // We have a synchronous error, we need to make sure to\n      // finalize right away. This ensures that callbacks in the `finalize` operator are called\n      // at the right time, and that finalization occurs at the expected\n      // time between the source error and the subscription to the\n      // next observable.\n      innerSub.unsubscribe();\n      innerSub = null;\n      handledResult!.subscribe(subscriber);\n    }\n  });\n}\n", "import { Observable } from '../Observable';\nimport { Subscriber } from '../Subscriber';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\n\n/**\n * A basic scan operation. This is used for `scan` and `reduce`.\n * @param accumulator The accumulator to use\n * @param seed The seed value for the state to accumulate\n * @param hasSeed Whether or not a seed was provided\n * @param emitOnNext Whether or not to emit the state on next\n * @param emitBeforeComplete Whether or not to emit the before completion\n */\n\nexport function scanInternals<V, A, S>(\n  accumulator: (acc: V | A | S, value: V, index: number) => A,\n  seed: S,\n  hasSeed: boolean,\n  emitOnNext: boolean,\n  emitBeforeComplete?: undefined | true\n) {\n  return (source: Observable<V>, subscriber: Subscriber<any>) => {\n    // Whether or not we have state yet. This will only be\n    // false before the first value arrives if we didn't get\n    // a seed value.\n    let hasState = hasSeed;\n    // The state that we're tracking, starting with the seed,\n    // if there is one, and then updated by the return value\n    // from the accumulator on each emission.\n    let state: any = seed;\n    // An index to pass to the accumulator function.\n    let index = 0;\n\n    // Subscribe to our source. All errors and completions are passed through.\n    source.subscribe(\n      createOperatorSubscriber(\n        subscriber,\n        (value) => {\n          // Always increment the index.\n          const i = index++;\n          // Set the state\n          state = hasState\n            ? // We already have state, so we can get the new state from the accumulator\n              accumulator(state, value, i)\n            : // We didn't have state yet, a seed value was not provided, so\n\n              // we set the state to the first value, and mark that we have state now\n              ((hasState = true), value);\n\n          // Maybe send it to the consumer.\n          emitOnNext && subscriber.next(state);\n        },\n        // If an onComplete was given, call it, otherwise\n        // just pass through the complete notification to the consumer.\n        emitBeforeComplete &&\n          (() => {\n            hasState && subscriber.next(state);\n            subscriber.complete();\n          })\n      )\n    );\n  };\n}\n", "import { combineLatestInit } from '../observable/combineLatest';\nimport { ObservableInput, ObservableInputTuple, OperatorFunction } from '../types';\nimport { operate } from '../util/lift';\nimport { argsOrArgArray } from '../util/argsOrArgArray';\nimport { mapOneOrManyArgs } from '../util/mapOneOrManyArgs';\nimport { pipe } from '../util/pipe';\nimport { popResultSelector } from '../util/args';\n\n/** @deprecated Replaced with {@link combineLatestWith}. Will be removed in v8. */\nexport function combineLatest<T, A extends readonly unknown[], R>(\n  sources: [...ObservableInputTuple<A>],\n  project: (...values: [T, ...A]) => R\n): OperatorFunction<T, R>;\n/** @deprecated Replaced with {@link combineLatestWith}. Will be removed in v8. */\nexport function combineLatest<T, A extends readonly unknown[], R>(sources: [...ObservableInputTuple<A>]): OperatorFunction<T, [T, ...A]>;\n\n/** @deprecated Replaced with {@link combineLatestWith}. Will be removed in v8. */\nexport function combineLatest<T, A extends readonly unknown[], R>(\n  ...sourcesAndProject: [...ObservableInputTuple<A>, (...values: [T, ...A]) => R]\n): OperatorFunction<T, R>;\n/** @deprecated Replaced with {@link combineLatestWith}. Will be removed in v8. */\nexport function combineLatest<T, A extends readonly unknown[], R>(...sources: [...ObservableInputTuple<A>]): OperatorFunction<T, [T, ...A]>;\n\n/**\n * @deprecated Replaced with {@link combineLatestWith}. Will be removed in v8.\n */\nexport function combineLatest<T, R>(...args: (ObservableInput<any> | ((...values: any[]) => R))[]): OperatorFunction<T, unknown> {\n  const resultSelector = popResultSelector(args);\n  return resultSelector\n    ? pipe(combineLatest(...(args as Array<ObservableInput<any>>)), mapOneOrManyArgs(resultSelector))\n    : operate((source, subscriber) => {\n        combineLatestInit([source, ...argsOrArgArray(args)])(subscriber);\n      });\n}\n", "import { ObservableInputTuple, OperatorFunction, Cons } from '../types';\nimport { combineLatest } from './combineLatest';\n\n/**\n * Create an observable that combines the latest values from all passed observables and the source\n * into arrays and emits them.\n *\n * Returns an observable, that when subscribed to, will subscribe to the source observable and all\n * sources provided as arguments. Once all sources emit at least one value, all of the latest values\n * will be emitted as an array. After that, every time any source emits a value, all of the latest values\n * will be emitted as an array.\n *\n * This is a useful operator for eagerly calculating values based off of changed inputs.\n *\n * ## Example\n *\n * Simple concatenation of values from two inputs\n *\n * ```ts\n * import { fromEvent, combineLatestWith, map } from 'rxjs';\n *\n * // Setup: Add two inputs to the page\n * const input1 = document.createElement('input');\n * document.body.appendChild(input1);\n * const input2 = document.createElement('input');\n * document.body.appendChild(input2);\n *\n * // Get streams of changes\n * const input1Changes$ = fromEvent(input1, 'change');\n * const input2Changes$ = fromEvent(input2, 'change');\n *\n * // Combine the changes by adding them together\n * input1Changes$.pipe(\n *   combineLatestWith(input2Changes$),\n *   map(([e1, e2]) => (<HTMLInputElement>e1.target).value + ' - ' + (<HTMLInputElement>e2.target).value)\n * )\n * .subscribe(x => console.log(x));\n * ```\n *\n * @param otherSources the other sources to subscribe to.\n * @return A function that returns an Observable that emits the latest\n * emissions from both source and provided Observables.\n */\nexport function combineLatestWith<T, A extends readonly unknown[]>(\n  ...otherSources: [...ObservableInputTuple<A>]\n): OperatorFunction<T, Cons<T, A>> {\n  return combineLatest(...otherSources);\n}\n", "import { asyncScheduler } from '../scheduler/async';\nimport { Subscription } from '../Subscription';\nimport { MonoTypeOperatorFunction, SchedulerAction, SchedulerLike } from '../types';\nimport { operate } from '../util/lift';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\n\n/**\n * Emits a notification from the source Observable only after a particular time span\n * has passed without another source emission.\n *\n * <span class=\"informal\">It's like {@link delay}, but passes only the most\n * recent notification from each burst of emissions.</span>\n *\n * ![](debounceTime.png)\n *\n * `debounceTime` delays notifications emitted by the source Observable, but drops\n * previous pending delayed emissions if a new notification arrives on the source\n * Observable. This operator keeps track of the most recent notification from the\n * source Observable, and emits that only when `dueTime` has passed\n * without any other notification appearing on the source Observable. If a new value\n * appears before `dueTime` silence occurs, the previous notification will be dropped\n * and will not be emitted and a new `dueTime` is scheduled.\n * If the completing event happens during `dueTime` the last cached notification\n * is emitted before the completion event is forwarded to the output observable.\n * If the error event happens during `dueTime` or after it only the error event is\n * forwarded to the output observable. The cache notification is not emitted in this case.\n *\n * This is a rate-limiting operator, because it is impossible for more than one\n * notification to be emitted in any time window of duration `dueTime`, but it is also\n * a delay-like operator since output emissions do not occur at the same time as\n * they did on the source Observable. Optionally takes a {@link SchedulerLike} for\n * managing timers.\n *\n * ## Example\n *\n * Emit the most recent click after a burst of clicks\n *\n * ```ts\n * import { fromEvent, debounceTime } from 'rxjs';\n *\n * const clicks = fromEvent(document, 'click');\n * const result = clicks.pipe(debounceTime(1000));\n * result.subscribe(x => console.log(x));\n * ```\n *\n * @see {@link audit}\n * @see {@link auditTime}\n * @see {@link debounce}\n * @see {@link sample}\n * @see {@link sampleTime}\n * @see {@link throttle}\n * @see {@link throttleTime}\n *\n * @param {number} dueTime The timeout duration in milliseconds (or the time\n * unit determined internally by the optional `scheduler`) for the window of\n * time required to wait for emission silence before emitting the most recent\n * source value.\n * @param {SchedulerLike} [scheduler=async] The {@link SchedulerLike} to use for\n * managing the timers that handle the timeout for each value.\n * @return A function that returns an Observable that delays the emissions of\n * the source Observable by the specified `dueTime`, and may drop some values\n * if they occur too frequently.\n */\nexport function debounceTime<T>(dueTime: number, scheduler: SchedulerLike = asyncScheduler): MonoTypeOperatorFunction<T> {\n  return operate((source, subscriber) => {\n    let activeTask: Subscription | null = null;\n    let lastValue: T | null = null;\n    let lastTime: number | null = null;\n\n    const emit = () => {\n      if (activeTask) {\n        // We have a value! Free up memory first, then emit the value.\n        activeTask.unsubscribe();\n        activeTask = null;\n        const value = lastValue!;\n        lastValue = null;\n        subscriber.next(value);\n      }\n    };\n    function emitWhenIdle(this: SchedulerAction<unknown>) {\n      // This is called `dueTime` after the first value\n      // but we might have received new values during this window!\n\n      const targetTime = lastTime! + dueTime;\n      const now = scheduler.now();\n      if (now < targetTime) {\n        // On that case, re-schedule to the new target\n        activeTask = this.schedule(undefined, targetTime - now);\n        subscriber.add(activeTask);\n        return;\n      }\n\n      emit();\n    }\n\n    source.subscribe(\n      createOperatorSubscriber(\n        subscriber,\n        (value: T) => {\n          lastValue = value;\n          lastTime = scheduler.now();\n\n          // Only set up a task if it's not already up\n          if (!activeTask) {\n            activeTask = scheduler.schedule(emitWhenIdle, dueTime);\n            subscriber.add(activeTask);\n          }\n        },\n        () => {\n          // Source completed.\n          // Emit any pending debounced values then complete\n          emit();\n          subscriber.complete();\n        },\n        // Pass all errors through to consumer.\n        undefined,\n        () => {\n          // Finalization.\n          lastValue = activeTask = null;\n        }\n      )\n    );\n  });\n}\n", "import { OperatorFunction } from '../types';\nimport { operate } from '../util/lift';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\n\n/**\n * Emits a given value if the source Observable completes without emitting any\n * `next` value, otherwise mirrors the source Observable.\n *\n * <span class=\"informal\">If the source Observable turns out to be empty, then\n * this operator will emit a default value.</span>\n *\n * ![](defaultIfEmpty.png)\n *\n * `defaultIfEmpty` emits the values emitted by the source Observable or a\n * specified default value if the source Observable is empty (completes without\n * having emitted any `next` value).\n *\n * ## Example\n *\n * If no clicks happen in 5 seconds, then emit 'no clicks'\n *\n * ```ts\n * import { fromEvent, takeUntil, interval, defaultIfEmpty } from 'rxjs';\n *\n * const clicks = fromEvent(document, 'click');\n * const clicksBeforeFive = clicks.pipe(takeUntil(interval(5000)));\n * const result = clicksBeforeFive.pipe(defaultIfEmpty('no clicks'));\n * result.subscribe(x => console.log(x));\n * ```\n *\n * @see {@link empty}\n * @see {@link last}\n *\n * @param defaultValue The default value used if the source\n * Observable is empty.\n * @return A function that returns an Observable that emits either the\n * specified `defaultValue` if the source Observable emits no items, or the\n * values emitted by the source Observable.\n */\nexport function defaultIfEmpty<T, R>(defaultValue: R): OperatorFunction<T, T | R> {\n  return operate((source, subscriber) => {\n    let hasValue = false;\n    source.subscribe(\n      createOperatorSubscriber(\n        subscriber,\n        (value) => {\n          hasValue = true;\n          subscriber.next(value);\n        },\n        () => {\n          if (!hasValue) {\n            subscriber.next(defaultValue!);\n          }\n          subscriber.complete();\n        }\n      )\n    );\n  });\n}\n", "import { MonoTypeOperatorFunction } from '../types';\nimport { EMPTY } from '../observable/empty';\nimport { operate } from '../util/lift';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\n\n/**\n * Emits only the first `count` values emitted by the source Observable.\n *\n * <span class=\"informal\">Takes the first `count` values from the source, then\n * completes.</span>\n *\n * ![](take.png)\n *\n * `take` returns an Observable that emits only the first `count` values emitted\n * by the source Observable. If the source emits fewer than `count` values then\n * all of its values are emitted. After that, it completes, regardless if the\n * source completes.\n *\n * ## Example\n *\n * Take the first 5 seconds of an infinite 1-second interval Observable\n *\n * ```ts\n * import { interval, take } from 'rxjs';\n *\n * const intervalCount = interval(1000);\n * const takeFive = intervalCount.pipe(take(5));\n * takeFive.subscribe(x => console.log(x));\n *\n * // Logs:\n * // 0\n * // 1\n * // 2\n * // 3\n * // 4\n * ```\n *\n * @see {@link takeLast}\n * @see {@link takeUntil}\n * @see {@link takeWhile}\n * @see {@link skip}\n *\n * @param count The maximum number of `next` values to emit.\n * @return A function that returns an Observable that emits only the first\n * `count` values emitted by the source Observable, or all of the values from\n * the source if the source emits fewer than `count` values.\n */\nexport function take<T>(count: number): MonoTypeOperatorFunction<T> {\n  return count <= 0\n    ? // If we are taking no values, that's empty.\n      () => EMPTY\n    : operate((source, subscriber) => {\n        let seen = 0;\n        source.subscribe(\n          createOperatorSubscriber(subscriber, (value) => {\n            // Increment the number of values we have seen,\n            // then check it against the allowed count to see\n            // if we are still letting values through.\n            if (++seen <= count) {\n              subscriber.next(value);\n              // If we have met or passed our allowed count,\n              // we need to complete. We have to do <= here,\n              // because re-entrant code will increment `seen` twice.\n              if (count <= seen) {\n                subscriber.complete();\n              }\n            }\n          })\n        );\n      });\n}\n", "import { OperatorFunction } from '../types';\nimport { operate } from '../util/lift';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\nimport { noop } from '../util/noop';\n\n/**\n * Ignores all items emitted by the source Observable and only passes calls of `complete` or `error`.\n *\n * ![](ignoreElements.png)\n *\n * The `ignoreElements` operator suppresses all items emitted by the source Observable,\n * but allows its termination notification (either `error` or `complete`) to pass through unchanged.\n *\n * If you do not care about the items being emitted by an Observable, but you do want to be notified\n * when it completes or when it terminates with an error, you can apply the `ignoreElements` operator\n * to the Observable, which will ensure that it will never call its observers\u2019 `next` handlers.\n *\n * ## Example\n *\n * Ignore all `next` emissions from the source\n *\n * ```ts\n * import { of, ignoreElements } from 'rxjs';\n *\n * of('you', 'talking', 'to', 'me')\n *   .pipe(ignoreElements())\n *   .subscribe({\n *     next: word => console.log(word),\n *     error: err => console.log('error:', err),\n *     complete: () => console.log('the end'),\n *   });\n *\n * // result:\n * // 'the end'\n * ```\n *\n * @return A function that returns an empty Observable that only calls\n * `complete` or `error`, based on which one is called by the source\n * Observable.\n */\nexport function ignoreElements(): OperatorFunction<unknown, never> {\n  return operate((source, subscriber) => {\n    source.subscribe(createOperatorSubscriber(subscriber, noop));\n  });\n}\n", "import { OperatorFunction } from '../types';\nimport { map } from './map';\n\n/** @deprecated To be removed in v9. Use {@link map} instead: `map(() => value)`. */\nexport function mapTo<R>(value: R): OperatorFunction<unknown, R>;\n/**\n * @deprecated Do not specify explicit type parameters. Signatures with type parameters\n * that cannot be inferred will be removed in v8. `mapTo` itself will be removed in v9,\n * use {@link map} instead: `map(() => value)`.\n * */\nexport function mapTo<T, R>(value: R): OperatorFunction<T, R>;\n\n/**\n * Emits the given constant value on the output Observable every time the source\n * Observable emits a value.\n *\n * <span class=\"informal\">Like {@link map}, but it maps every source value to\n * the same output value every time.</span>\n *\n * ![](mapTo.png)\n *\n * Takes a constant `value` as argument, and emits that whenever the source\n * Observable emits a value. In other words, ignores the actual source value,\n * and simply uses the emission moment to know when to emit the given `value`.\n *\n * ## Example\n *\n * Map every click to the string `'Hi'`\n *\n * ```ts\n * import { fromEvent, mapTo } from 'rxjs';\n *\n * const clicks = fromEvent(document, 'click');\n * const greetings = clicks.pipe(mapTo('Hi'));\n *\n * greetings.subscribe(x => console.log(x));\n * ```\n *\n * @see {@link map}\n *\n * @param value The value to map each source value to.\n * @return A function that returns an Observable that emits the given `value`\n * every time the source Observable emits.\n * @deprecated To be removed in v9. Use {@link map} instead: `map(() => value)`.\n */\nexport function mapTo<R>(value: R): OperatorFunction<unknown, R> {\n  return map(() => value);\n}\n", "import { Observable } from '../Observable';\nimport { MonoTypeOperatorFunction, ObservableInput } from '../types';\nimport { concat } from '../observable/concat';\nimport { take } from './take';\nimport { ignoreElements } from './ignoreElements';\nimport { mapTo } from './mapTo';\nimport { mergeMap } from './mergeMap';\nimport { innerFrom } from '../observable/innerFrom';\n\n/** @deprecated The `subscriptionDelay` parameter will be removed in v8. */\nexport function delayWhen<T>(\n  delayDurationSelector: (value: T, index: number) => ObservableInput<any>,\n  subscriptionDelay: Observable<any>\n): MonoTypeOperatorFunction<T>;\nexport function delayWhen<T>(delayDurationSelector: (value: T, index: number) => ObservableInput<any>): MonoTypeOperatorFunction<T>;\n\n/**\n * Delays the emission of items from the source Observable by a given time span\n * determined by the emissions of another Observable.\n *\n * <span class=\"informal\">It's like {@link delay}, but the time span of the\n * delay duration is determined by a second Observable.</span>\n *\n * ![](delayWhen.png)\n *\n * `delayWhen` operator shifts each emitted value from the source Observable by\n * a time span determined by another Observable. When the source emits a value,\n * the `delayDurationSelector` function is called with the value emitted from\n * the source Observable as the first argument to the `delayDurationSelector`.\n * The `delayDurationSelector` function should return an {@link ObservableInput},\n * that is internally converted to an Observable that is called the \"duration\"\n * Observable.\n *\n * The source value is emitted on the output Observable only when the \"duration\"\n * Observable emits ({@link guide/glossary-and-semantics#next next}s) any value.\n * Upon that, the \"duration\" Observable gets unsubscribed.\n *\n * Before RxJS V7, the {@link guide/glossary-and-semantics#complete completion}\n * of the \"duration\" Observable would have been triggering the emission of the\n * source value to the output Observable, but with RxJS V7, this is not the case\n * anymore.\n *\n * Only next notifications (from the \"duration\" Observable) trigger values from\n * the source Observable to be passed to the output Observable. If the \"duration\"\n * Observable only emits the complete notification (without next), the value\n * emitted by the source Observable will never get to the output Observable - it\n * will be swallowed. If the \"duration\" Observable errors, the error will be\n * propagated to the output Observable.\n *\n * Optionally, `delayWhen` takes a second argument, `subscriptionDelay`, which\n * is an Observable. When `subscriptionDelay` emits its first value or\n * completes, the source Observable is subscribed to and starts behaving like\n * described in the previous paragraph. If `subscriptionDelay` is not provided,\n * `delayWhen` will subscribe to the source Observable as soon as the output\n * Observable is subscribed.\n *\n * ## Example\n *\n * Delay each click by a random amount of time, between 0 and 5 seconds\n *\n * ```ts\n * import { fromEvent, delayWhen, interval } from 'rxjs';\n *\n * const clicks = fromEvent(document, 'click');\n * const delayedClicks = clicks.pipe(\n *   delayWhen(() => interval(Math.random() * 5000))\n * );\n * delayedClicks.subscribe(x => console.log(x));\n * ```\n *\n * @see {@link delay}\n * @see {@link throttle}\n * @see {@link throttleTime}\n * @see {@link debounce}\n * @see {@link debounceTime}\n * @see {@link sample}\n * @see {@link sampleTime}\n * @see {@link audit}\n * @see {@link auditTime}\n *\n * @param delayDurationSelector A function that returns an `ObservableInput` for\n * each `value` emitted by the source Observable, which is then used to delay the\n * emission of that `value` on the output Observable until the `ObservableInput`\n * returned from this function emits a next value. When called, beside `value`,\n * this function receives a zero-based `index` of the emission order.\n * @param subscriptionDelay An Observable that triggers the subscription to the\n * source Observable once it emits any value.\n * @return A function that returns an Observable that delays the emissions of\n * the source Observable by an amount of time specified by the Observable\n * returned by `delayDurationSelector`.\n */\nexport function delayWhen<T>(\n  delayDurationSelector: (value: T, index: number) => ObservableInput<any>,\n  subscriptionDelay?: Observable<any>\n): MonoTypeOperatorFunction<T> {\n  if (subscriptionDelay) {\n    // DEPRECATED PATH\n    return (source: Observable<T>) =>\n      concat(subscriptionDelay.pipe(take(1), ignoreElements()), source.pipe(delayWhen(delayDurationSelector)));\n  }\n\n  return mergeMap((value, index) => innerFrom(delayDurationSelector(value, index)).pipe(take(1), mapTo(value)));\n}\n", "import { asyncScheduler } from '../scheduler/async';\nimport { MonoTypeOperatorFunction, SchedulerLike } from '../types';\nimport { delayWhen } from './delayWhen';\nimport { timer } from '../observable/timer';\n\n/**\n * Delays the emission of items from the source Observable by a given timeout or\n * until a given Date.\n *\n * <span class=\"informal\">Time shifts each item by some specified amount of\n * milliseconds.</span>\n *\n * ![](delay.svg)\n *\n * If the delay argument is a Number, this operator time shifts the source\n * Observable by that amount of time expressed in milliseconds. The relative\n * time intervals between the values are preserved.\n *\n * If the delay argument is a Date, this operator time shifts the start of the\n * Observable execution until the given date occurs.\n *\n * ## Examples\n *\n * Delay each click by one second\n *\n * ```ts\n * import { fromEvent, delay } from 'rxjs';\n *\n * const clicks = fromEvent(document, 'click');\n * const delayedClicks = clicks.pipe(delay(1000)); // each click emitted after 1 second\n * delayedClicks.subscribe(x => console.log(x));\n * ```\n *\n * Delay all clicks until a future date happens\n *\n * ```ts\n * import { fromEvent, delay } from 'rxjs';\n *\n * const clicks = fromEvent(document, 'click');\n * const date = new Date('March 15, 2050 12:00:00'); // in the future\n * const delayedClicks = clicks.pipe(delay(date)); // click emitted only after that date\n * delayedClicks.subscribe(x => console.log(x));\n * ```\n *\n * @see {@link delayWhen}\n * @see {@link throttle}\n * @see {@link throttleTime}\n * @see {@link debounce}\n * @see {@link debounceTime}\n * @see {@link sample}\n * @see {@link sampleTime}\n * @see {@link audit}\n * @see {@link auditTime}\n *\n * @param {number|Date} due The delay duration in milliseconds (a `number`) or\n * a `Date` until which the emission of the source items is delayed.\n * @param {SchedulerLike} [scheduler=async] The {@link SchedulerLike} to use for\n * managing the timers that handle the time-shift for each item.\n * @return A function that returns an Observable that delays the emissions of\n * the source Observable by the specified timeout or Date.\n */\nexport function delay<T>(due: number | Date, scheduler: SchedulerLike = asyncScheduler): MonoTypeOperatorFunction<T> {\n  const duration = timer(due, scheduler);\n  return delayWhen(() => duration);\n}\n", "import { MonoTypeOperatorFunction } from '../types';\nimport { identity } from '../util/identity';\nimport { operate } from '../util/lift';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\n\nexport function distinctUntilChanged<T>(comparator?: (previous: T, current: T) => boolean): MonoTypeOperatorFunction<T>;\nexport function distinctUntilChanged<T, K>(\n  comparator: (previous: K, current: K) => boolean,\n  keySelector: (value: T) => K\n): MonoTypeOperatorFunction<T>;\n\n/**\n * Returns a result {@link Observable} that emits all values pushed by the source observable if they\n * are distinct in comparison to the last value the result observable emitted.\n *\n * When provided without parameters or with the first parameter (`{@link distinctUntilChanged#comparator comparator}`),\n * it behaves like this:\n *\n * 1. It will always emit the first value from the source.\n * 2. For all subsequent values pushed by the source, they will be compared to the previously emitted values\n *    using the provided `comparator` or an `===` equality check.\n * 3. If the value pushed by the source is determined to be unequal by this check, that value is emitted and\n *    becomes the new \"previously emitted value\" internally.\n *\n * When the second parameter (`{@link distinctUntilChanged#keySelector keySelector}`) is provided, the behavior\n * changes:\n *\n * 1. It will always emit the first value from the source.\n * 2. The `keySelector` will be run against all values, including the first value.\n * 3. For all values after the first, the selected key will be compared against the key selected from\n *    the previously emitted value using the `comparator`.\n * 4. If the keys are determined to be unequal by this check, the value (not the key), is emitted\n *    and the selected key from that value is saved for future comparisons against other keys.\n *\n * ## Examples\n *\n * A very basic example with no `{@link distinctUntilChanged#comparator comparator}`. Note that `1` is emitted more than once,\n * because it's distinct in comparison to the _previously emitted_ value,\n * not in comparison to _all other emitted values_.\n *\n * ```ts\n * import { of, distinctUntilChanged } from 'rxjs';\n *\n * of(1, 1, 1, 2, 2, 2, 1, 1, 3, 3)\n *   .pipe(distinctUntilChanged())\n *   .subscribe(console.log);\n * // Logs: 1, 2, 1, 3\n * ```\n *\n * With a `{@link distinctUntilChanged#comparator comparator}`, you can do custom comparisons. Let's say\n * you only want to emit a value when all of its components have\n * changed:\n *\n * ```ts\n * import { of, distinctUntilChanged } from 'rxjs';\n *\n * const totallyDifferentBuilds$ = of(\n *   { engineVersion: '1.1.0', transmissionVersion: '1.2.0' },\n *   { engineVersion: '1.1.0', transmissionVersion: '1.4.0' },\n *   { engineVersion: '1.3.0', transmissionVersion: '1.4.0' },\n *   { engineVersion: '1.3.0', transmissionVersion: '1.5.0' },\n *   { engineVersion: '2.0.0', transmissionVersion: '1.5.0' }\n * ).pipe(\n *   distinctUntilChanged((prev, curr) => {\n *     return (\n *       prev.engineVersion === curr.engineVersion ||\n *       prev.transmissionVersion === curr.transmissionVersion\n *     );\n *   })\n * );\n *\n * totallyDifferentBuilds$.subscribe(console.log);\n *\n * // Logs:\n * // { engineVersion: '1.1.0', transmissionVersion: '1.2.0' }\n * // { engineVersion: '1.3.0', transmissionVersion: '1.4.0' }\n * // { engineVersion: '2.0.0', transmissionVersion: '1.5.0' }\n * ```\n *\n * You can also provide a custom `{@link distinctUntilChanged#comparator comparator}` to check that emitted\n * changes are only in one direction. Let's say you only want to get\n * the next record temperature:\n *\n * ```ts\n * import { of, distinctUntilChanged } from 'rxjs';\n *\n * const temps$ = of(30, 31, 20, 34, 33, 29, 35, 20);\n *\n * const recordHighs$ = temps$.pipe(\n *   distinctUntilChanged((prevHigh, temp) => {\n *     // If the current temp is less than\n *     // or the same as the previous record,\n *     // the record hasn't changed.\n *     return temp <= prevHigh;\n *   })\n * );\n *\n * recordHighs$.subscribe(console.log);\n * // Logs: 30, 31, 34, 35\n * ```\n *\n * Selecting update events only when the `updatedBy` field shows\n * the account changed hands.\n *\n * ```ts\n * import { of, distinctUntilChanged } from 'rxjs';\n *\n * // A stream of updates to a given account\n * const accountUpdates$ = of(\n *   { updatedBy: 'blesh', data: [] },\n *   { updatedBy: 'blesh', data: [] },\n *   { updatedBy: 'ncjamieson', data: [] },\n *   { updatedBy: 'ncjamieson', data: [] },\n *   { updatedBy: 'blesh', data: [] }\n * );\n *\n * // We only want the events where it changed hands\n * const changedHands$ = accountUpdates$.pipe(\n *   distinctUntilChanged(undefined, update => update.updatedBy)\n * );\n *\n * changedHands$.subscribe(console.log);\n * // Logs:\n * // { updatedBy: 'blesh', data: Array[0] }\n * // { updatedBy: 'ncjamieson', data: Array[0] }\n * // { updatedBy: 'blesh', data: Array[0] }\n * ```\n *\n * @see {@link distinct}\n * @see {@link distinctUntilKeyChanged}\n *\n * @param comparator A function used to compare the previous and current keys for\n * equality. Defaults to a `===` check.\n * @param keySelector Used to select a key value to be passed to the `comparator`.\n *\n * @return A function that returns an Observable that emits items from the\n * source Observable with distinct values.\n */\nexport function distinctUntilChanged<T, K>(\n  comparator?: (previous: K, current: K) => boolean,\n  keySelector: (value: T) => K = identity as (value: T) => K\n): MonoTypeOperatorFunction<T> {\n  // We've been allowing `null` do be passed as the `compare`, so we can't do\n  // a default value for the parameter, because that will only work\n  // for `undefined`.\n  comparator = comparator ?? defaultCompare;\n\n  return operate((source, subscriber) => {\n    // The previous key, used to compare against keys selected\n    // from new arrivals to determine \"distinctiveness\".\n    let previousKey: K;\n    // Whether or not this is the first value we've gotten.\n    let first = true;\n\n    source.subscribe(\n      createOperatorSubscriber(subscriber, (value) => {\n        // We always call the key selector.\n        const currentKey = keySelector(value);\n\n        // If it's the first value, we always emit it.\n        // Otherwise, we compare this key to the previous key, and\n        // if the comparer returns false, we emit.\n        if (first || !comparator!(previousKey, currentKey)) {\n          // Update our state *before* we emit the value\n          // as emission can be the source of re-entrant code\n          // in functional libraries like this. We only really\n          // need to do this if it's the first value, or if the\n          // key we're tracking in previous needs to change.\n          first = false;\n          previousKey = currentKey;\n\n          // Emit the value!\n          subscriber.next(value);\n        }\n      })\n    );\n  });\n}\n\nfunction defaultCompare(a: any, b: any) {\n  return a === b;\n}\n", "import { distinctUntilChanged } from './distinctUntilChanged';\nimport { MonoTypeOperatorFunction } from '../types';\n\n/* tslint:disable:max-line-length */\nexport function distinctUntilKeyChanged<T>(key: keyof T): MonoTypeOperatorFunction<T>;\nexport function distinctUntilKeyChanged<T, K extends keyof T>(key: K, compare: (x: T[K], y: T[K]) => boolean): MonoTypeOperatorFunction<T>;\n/* tslint:enable:max-line-length */\n\n/**\n * Returns an Observable that emits all items emitted by the source Observable that are distinct by comparison from the previous item,\n * using a property accessed by using the key provided to check if the two items are distinct.\n *\n * If a comparator function is provided, then it will be called for each item to test for whether or not that value should be emitted.\n *\n * If a comparator function is not provided, an equality check is used by default.\n *\n * ## Examples\n *\n * An example comparing the name of persons\n *\n * ```ts\n * import { of, distinctUntilKeyChanged } from 'rxjs';\n *\n * of(\n *   { age: 4, name: 'Foo' },\n *   { age: 7, name: 'Bar' },\n *   { age: 5, name: 'Foo' },\n *   { age: 6, name: 'Foo' }\n * ).pipe(\n *   distinctUntilKeyChanged('name')\n * )\n * .subscribe(x => console.log(x));\n *\n * // displays:\n * // { age: 4, name: 'Foo' }\n * // { age: 7, name: 'Bar' }\n * // { age: 5, name: 'Foo' }\n * ```\n *\n * An example comparing the first letters of the name\n *\n * ```ts\n * import { of, distinctUntilKeyChanged } from 'rxjs';\n *\n * of(\n *   { age: 4, name: 'Foo1' },\n *   { age: 7, name: 'Bar' },\n *   { age: 5, name: 'Foo2' },\n *   { age: 6, name: 'Foo3' }\n * ).pipe(\n *   distinctUntilKeyChanged('name', (x, y) => x.substring(0, 3) === y.substring(0, 3))\n * )\n * .subscribe(x => console.log(x));\n *\n * // displays:\n * // { age: 4, name: 'Foo1' }\n * // { age: 7, name: 'Bar' }\n * // { age: 5, name: 'Foo2' }\n * ```\n *\n * @see {@link distinct}\n * @see {@link distinctUntilChanged}\n *\n * @param {string} key String key for object property lookup on each item.\n * @param {function} [compare] Optional comparison function called to test if an item is distinct from the previous item in the source.\n * @return A function that returns an Observable that emits items from the\n * source Observable with distinct values based on the key specified.\n */\nexport function distinctUntilKeyChanged<T, K extends keyof T>(key: K, compare?: (x: T[K], y: T[K]) => boolean): MonoTypeOperatorFunction<T> {\n  return distinctUntilChanged((x: T, y: T) => compare ? compare(x[key], y[key]) : x[key] === y[key]);\n}\n", "import { EmptyError } from '../util/EmptyError';\nimport { MonoTypeOperatorFunction } from '../types';\nimport { operate } from '../util/lift';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\n\n/**\n * If the source observable completes without emitting a value, it will emit\n * an error. The error will be created at that time by the optional\n * `errorFactory` argument, otherwise, the error will be {@link EmptyError}.\n *\n * ![](throwIfEmpty.png)\n *\n * ## Example\n *\n * Throw an error if the document wasn't clicked within 1 second\n *\n * ```ts\n * import { fromEvent, takeUntil, timer, throwIfEmpty } from 'rxjs';\n *\n * const click$ = fromEvent(document, 'click');\n *\n * click$.pipe(\n *   takeUntil(timer(1000)),\n *   throwIfEmpty(() => new Error('The document was not clicked within 1 second'))\n * )\n * .subscribe({\n *   next() {\n *    console.log('The document was clicked');\n *   },\n *   error(err) {\n *     console.error(err.message);\n *   }\n * });\n * ```\n *\n * @param errorFactory A factory function called to produce the\n * error to be thrown when the source observable completes without emitting a\n * value.\n * @return A function that returns an Observable that throws an error if the\n * source Observable completed without emitting.\n */\nexport function throwIfEmpty<T>(errorFactory: () => any = defaultErrorFactory): MonoTypeOperatorFunction<T> {\n  return operate((source, subscriber) => {\n    let hasValue = false;\n    source.subscribe(\n      createOperatorSubscriber(\n        subscriber,\n        (value) => {\n          hasValue = true;\n          subscriber.next(value);\n        },\n        () => (hasValue ? subscriber.complete() : subscriber.error(errorFactory()))\n      )\n    );\n  });\n}\n\nfunction defaultErrorFactory() {\n  return new EmptyError();\n}\n", "/** prettier */\nimport { Observable } from '../Observable';\nimport { concat } from '../observable/concat';\nimport { of } from '../observable/of';\nimport { MonoTypeOperatorFunction, SchedulerLike, OperatorFunction, ValueFromArray } from '../types';\n\n/** @deprecated The `scheduler` parameter will be removed in v8. Use `scheduled` and `concatAll`. Details: https://rxjs.dev/deprecations/scheduler-argument */\nexport function endWith<T>(scheduler: SchedulerLike): MonoTypeOperatorFunction<T>;\n/** @deprecated The `scheduler` parameter will be removed in v8. Use `scheduled` and `concatAll`. Details: https://rxjs.dev/deprecations/scheduler-argument */\nexport function endWith<T, A extends unknown[] = T[]>(\n  ...valuesAndScheduler: [...A, SchedulerLike]\n): OperatorFunction<T, T | ValueFromArray<A>>;\n\nexport function endWith<T, A extends unknown[] = T[]>(...values: A): OperatorFunction<T, T | ValueFromArray<A>>;\n\n/**\n * Returns an observable that will emit all values from the source, then synchronously emit\n * the provided value(s) immediately after the source completes.\n *\n * NOTE: Passing a last argument of a Scheduler is _deprecated_, and may result in incorrect\n * types in TypeScript.\n *\n * This is useful for knowing when an observable ends. Particularly when paired with an\n * operator like {@link takeUntil}\n *\n * ![](endWith.png)\n *\n * ## Example\n *\n * Emit values to know when an interval starts and stops. The interval will\n * stop when a user clicks anywhere on the document.\n *\n * ```ts\n * import { interval, map, fromEvent, startWith, takeUntil, endWith } from 'rxjs';\n *\n * const ticker$ = interval(5000).pipe(\n *   map(() => 'tick')\n * );\n *\n * const documentClicks$ = fromEvent(document, 'click');\n *\n * ticker$.pipe(\n *   startWith('interval started'),\n *   takeUntil(documentClicks$),\n *   endWith('interval ended by click')\n * )\n * .subscribe(x => console.log(x));\n *\n * // Result (assuming a user clicks after 15 seconds)\n * // 'interval started'\n * // 'tick'\n * // 'tick'\n * // 'tick'\n * // 'interval ended by click'\n * ```\n *\n * @see {@link startWith}\n * @see {@link concat}\n * @see {@link takeUntil}\n *\n * @param values Items you want the modified Observable to emit last.\n * @return A function that returns an Observable that emits all values from the\n * source, then synchronously emits the provided value(s) immediately after the\n * source completes.\n */\nexport function endWith<T>(...values: Array<T | SchedulerLike>): MonoTypeOperatorFunction<T> {\n  return (source: Observable<T>) => concat(source, of(...values)) as Observable<T>;\n}\n", "import { MonoTypeOperatorFunction } from '../types';\nimport { operate } from '../util/lift';\n\n/**\n * Returns an Observable that mirrors the source Observable, but will call a specified function when\n * the source terminates on complete or error.\n * The specified function will also be called when the subscriber explicitly unsubscribes.\n *\n * ## Examples\n *\n * Execute callback function when the observable completes\n *\n * ```ts\n * import { interval, take, finalize } from 'rxjs';\n *\n * // emit value in sequence every 1 second\n * const source = interval(1000);\n * const example = source.pipe(\n *   take(5), //take only the first 5 values\n *   finalize(() => console.log('Sequence complete')) // Execute when the observable completes\n * );\n * const subscribe = example.subscribe(val => console.log(val));\n *\n * // results:\n * // 0\n * // 1\n * // 2\n * // 3\n * // 4\n * // 'Sequence complete'\n * ```\n *\n * Execute callback function when the subscriber explicitly unsubscribes\n *\n * ```ts\n * import { interval, finalize, tap, noop, timer } from 'rxjs';\n *\n * const source = interval(100).pipe(\n *   finalize(() => console.log('[finalize] Called')),\n *   tap({\n *     next: () => console.log('[next] Called'),\n *     error: () => console.log('[error] Not called'),\n *     complete: () => console.log('[tap complete] Not called')\n *   })\n * );\n *\n * const sub = source.subscribe({\n *   next: x => console.log(x),\n *   error: noop,\n *   complete: () => console.log('[complete] Not called')\n * });\n *\n * timer(150).subscribe(() => sub.unsubscribe());\n *\n * // results:\n * // '[next] Called'\n * // 0\n * // '[finalize] Called'\n * ```\n *\n * @param {function} callback Function to be called when source terminates.\n * @return A function that returns an Observable that mirrors the source, but\n * will call the specified function on termination.\n */\nexport function finalize<T>(callback: () => void): MonoTypeOperatorFunction<T> {\n  return operate((source, subscriber) => {\n    // TODO: This try/finally was only added for `useDeprecatedSynchronousErrorHandling`.\n    // REMOVE THIS WHEN THAT HOT GARBAGE IS REMOVED IN V8.\n    try {\n      source.subscribe(subscriber);\n    } finally {\n      subscriber.add(callback);\n    }\n  });\n}\n", "import { Observable } from '../Observable';\nimport { EmptyError } from '../util/EmptyError';\nimport { OperatorFunction, TruthyTypesOf } from '../types';\nimport { filter } from './filter';\nimport { take } from './take';\nimport { defaultIfEmpty } from './defaultIfEmpty';\nimport { throwIfEmpty } from './throwIfEmpty';\nimport { identity } from '../util/identity';\n\nexport function first<T, D = T>(predicate?: null, defaultValue?: D): OperatorFunction<T, T | D>;\nexport function first<T>(predicate: BooleanConstructor): OperatorFunction<T, TruthyTypesOf<T>>;\nexport function first<T, D>(predicate: BooleanConstructor, defaultValue: D): OperatorFunction<T, TruthyTypesOf<T> | D>;\nexport function first<T, S extends T>(\n  predicate: (value: T, index: number, source: Observable<T>) => value is S,\n  defaultValue?: S\n): OperatorFunction<T, S>;\nexport function first<T, S extends T, D>(\n  predicate: (value: T, index: number, source: Observable<T>) => value is S,\n  defaultValue: D\n): OperatorFunction<T, S | D>;\nexport function first<T, D = T>(\n  predicate: (value: T, index: number, source: Observable<T>) => boolean,\n  defaultValue?: D\n): OperatorFunction<T, T | D>;\n\n/**\n * Emits only the first value (or the first value that meets some condition)\n * emitted by the source Observable.\n *\n * <span class=\"informal\">Emits only the first value. Or emits only the first\n * value that passes some test.</span>\n *\n * ![](first.png)\n *\n * If called with no arguments, `first` emits the first value of the source\n * Observable, then completes. If called with a `predicate` function, `first`\n * emits the first value of the source that matches the specified condition. Throws an error if\n * `defaultValue` was not provided and a matching element is not found.\n *\n * ## Examples\n *\n * Emit only the first click that happens on the DOM\n *\n * ```ts\n * import { fromEvent, first } from 'rxjs';\n *\n * const clicks = fromEvent(document, 'click');\n * const result = clicks.pipe(first());\n * result.subscribe(x => console.log(x));\n * ```\n *\n * Emits the first click that happens on a DIV\n *\n * ```ts\n * import { fromEvent, first } from 'rxjs';\n *\n * const div = document.createElement('div');\n * div.style.cssText = 'width: 200px; height: 200px; background: #09c;';\n * document.body.appendChild(div);\n *\n * const clicks = fromEvent(document, 'click');\n * const result = clicks.pipe(first(ev => (<HTMLElement>ev.target).tagName === 'DIV'));\n * result.subscribe(x => console.log(x));\n * ```\n *\n * @see {@link filter}\n * @see {@link find}\n * @see {@link take}\n *\n * @throws {EmptyError} Delivers an EmptyError to the Observer's `error`\n * callback if the Observable completes before any `next` notification was sent.\n * This is how `first()` is different from {@link take}(1) which completes instead.\n *\n * @param {function(value: T, index: number, source: Observable<T>): boolean} [predicate]\n * An optional function called with each item to test for condition matching.\n * @param {D} [defaultValue] The default value emitted in case no valid value\n * was found on the source.\n * @return A function that returns an Observable that emits the first item that\n * matches the condition.\n */\nexport function first<T, D>(\n  predicate?: ((value: T, index: number, source: Observable<T>) => boolean) | null,\n  defaultValue?: D\n): OperatorFunction<T, T | D> {\n  const hasDefaultValue = arguments.length >= 2;\n  return (source: Observable<T>) =>\n    source.pipe(\n      predicate ? filter((v, i) => predicate(v, i, source)) : identity,\n      take(1),\n      hasDefaultValue ? defaultIfEmpty(defaultValue!) : throwIfEmpty(() => new EmptyError())\n    );\n}\n", "import { EMPTY } from '../observable/empty';\nimport { MonoTypeOperatorFunction } from '../types';\nimport { operate } from '../util/lift';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\n\n/**\n * Waits for the source to complete, then emits the last N values from the source,\n * as specified by the `count` argument.\n *\n * ![](takeLast.png)\n *\n * `takeLast` results in an observable that will hold values up to `count` values in memory,\n * until the source completes. It then pushes all values in memory to the consumer, in the\n * order they were received from the source, then notifies the consumer that it is\n * complete.\n *\n * If for some reason the source completes before the `count` supplied to `takeLast` is reached,\n * all values received until that point are emitted, and then completion is notified.\n *\n * **Warning**: Using `takeLast` with an observable that never completes will result\n * in an observable that never emits a value.\n *\n * ## Example\n *\n * Take the last 3 values of an Observable with many values\n *\n * ```ts\n * import { range, takeLast } from 'rxjs';\n *\n * const many = range(1, 100);\n * const lastThree = many.pipe(takeLast(3));\n * lastThree.subscribe(x => console.log(x));\n * ```\n *\n * @see {@link take}\n * @see {@link takeUntil}\n * @see {@link takeWhile}\n * @see {@link skip}\n *\n * @param count The maximum number of values to emit from the end of\n * the sequence of values emitted by the source Observable.\n * @return A function that returns an Observable that emits at most the last\n * `count` values emitted by the source Observable.\n */\nexport function takeLast<T>(count: number): MonoTypeOperatorFunction<T> {\n  return count <= 0\n    ? () => EMPTY\n    : operate((source, subscriber) => {\n        // This buffer will hold the values we are going to emit\n        // when the source completes. Since we only want to take the\n        // last N values, we can't emit until we're sure we're not getting\n        // any more values.\n        let buffer: T[] = [];\n        source.subscribe(\n          createOperatorSubscriber(\n            subscriber,\n            (value) => {\n              // Add the most recent value onto the end of our buffer.\n              buffer.push(value);\n              // If our buffer is now larger than the number of values we\n              // want to take, we remove the oldest value from the buffer.\n              count < buffer.length && buffer.shift();\n            },\n            () => {\n              // The source completed, we now know what are last values\n              // are, emit them in the order they were received.\n              for (const value of buffer) {\n                subscriber.next(value);\n              }\n              subscriber.complete();\n            },\n            // Errors are passed through to the consumer\n            undefined,\n            () => {\n              // During finalization release the values in our buffer.\n              buffer = null!;\n            }\n          )\n        );\n      });\n}\n", "import { ObservableInput, ObservableInputTuple, OperatorFunction, SchedulerLike } from '../types';\nimport { operate } from '../util/lift';\nimport { argsOrArgArray } from '../util/argsOrArgArray';\nimport { mergeAll } from './mergeAll';\nimport { popNumber, popScheduler } from '../util/args';\nimport { from } from '../observable/from';\n\n/** @deprecated Replaced with {@link mergeWith}. Will be removed in v8. */\nexport function merge<T, A extends readonly unknown[]>(...sources: [...ObservableInputTuple<A>]): OperatorFunction<T, T | A[number]>;\n/** @deprecated Replaced with {@link mergeWith}. Will be removed in v8. */\nexport function merge<T, A extends readonly unknown[]>(\n  ...sourcesAndConcurrency: [...ObservableInputTuple<A>, number]\n): OperatorFunction<T, T | A[number]>;\n/** @deprecated Replaced with {@link mergeWith}. Will be removed in v8. */\nexport function merge<T, A extends readonly unknown[]>(\n  ...sourcesAndScheduler: [...ObservableInputTuple<A>, SchedulerLike]\n): OperatorFunction<T, T | A[number]>;\n/** @deprecated Replaced with {@link mergeWith}. Will be removed in v8. */\nexport function merge<T, A extends readonly unknown[]>(\n  ...sourcesAndConcurrencyAndScheduler: [...ObservableInputTuple<A>, number, SchedulerLike]\n): OperatorFunction<T, T | A[number]>;\n\nexport function merge<T>(...args: unknown[]): OperatorFunction<T, unknown> {\n  const scheduler = popScheduler(args);\n  const concurrent = popNumber(args, Infinity);\n  args = argsOrArgArray(args);\n\n  return operate((source, subscriber) => {\n    mergeAll(concurrent)(from([source, ...(args as ObservableInput<T>[])], scheduler)).subscribe(subscriber);\n  });\n}\n", "import { ObservableInputTuple, OperatorFunction } from '../types';\nimport { merge } from './merge';\n\n/**\n * Merge the values from all observables to a single observable result.\n *\n * Creates an observable, that when subscribed to, subscribes to the source\n * observable, and all other sources provided as arguments. All values from\n * every source are emitted from the resulting subscription.\n *\n * When all sources complete, the resulting observable will complete.\n *\n * When any source errors, the resulting observable will error.\n *\n * ## Example\n *\n * Joining all outputs from multiple user input event streams\n *\n * ```ts\n * import { fromEvent, map, mergeWith } from 'rxjs';\n *\n * const clicks$ = fromEvent(document, 'click').pipe(map(() => 'click'));\n * const mousemoves$ = fromEvent(document, 'mousemove').pipe(map(() => 'mousemove'));\n * const dblclicks$ = fromEvent(document, 'dblclick').pipe(map(() => 'dblclick'));\n *\n * mousemoves$\n *   .pipe(mergeWith(clicks$, dblclicks$))\n *   .subscribe(x => console.log(x));\n *\n * // result (assuming user interactions)\n * // 'mousemove'\n * // 'mousemove'\n * // 'mousemove'\n * // 'click'\n * // 'click'\n * // 'dblclick'\n * ```\n *\n * @see {@link merge}\n *\n * @param otherSources the sources to combine the current source with.\n * @return A function that returns an Observable that merges the values from\n * all given Observables.\n */\nexport function mergeWith<T, A extends readonly unknown[]>(\n  ...otherSources: [...ObservableInputTuple<A>]\n): OperatorFunction<T, T | A[number]> {\n  return merge(...otherSources);\n}\n", "import { Subscription } from '../Subscription';\nimport { EMPTY } from '../observable/empty';\nimport { operate } from '../util/lift';\nimport { MonoTypeOperatorFunction, ObservableInput } from '../types';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\nimport { innerFrom } from '../observable/innerFrom';\nimport { timer } from '../observable/timer';\n\nexport interface RepeatConfig {\n  /**\n   * The number of times to repeat the source. Defaults to `Infinity`.\n   */\n  count?: number;\n\n  /**\n   * If a `number`, will delay the repeat of the source by that number of milliseconds.\n   * If a function, it will provide the number of times the source has been subscribed to,\n   * and the return value should be a valid observable input that will notify when the source\n   * should be repeated. If the notifier observable is empty, the result will complete.\n   */\n  delay?: number | ((count: number) => ObservableInput<any>);\n}\n\n/**\n * Returns an Observable that will resubscribe to the source stream when the source stream completes.\n *\n * <span class=\"informal\">Repeats all values emitted on the source. It's like {@link retry}, but for non error cases.</span>\n *\n * ![](repeat.png)\n *\n * Repeat will output values from a source until the source completes, then it will resubscribe to the\n * source a specified number of times, with a specified delay. Repeat can be particularly useful in\n * combination with closing operators like {@link take}, {@link takeUntil}, {@link first}, or {@link takeWhile},\n * as it can be used to restart a source again from scratch.\n *\n * Repeat is very similar to {@link retry}, where {@link retry} will resubscribe to the source in the error case, but\n * `repeat` will resubscribe if the source completes.\n *\n * Note that `repeat` will _not_ catch errors. Use {@link retry} for that.\n *\n * - `repeat(0)` returns an empty observable\n * - `repeat()` will repeat forever\n * - `repeat({ delay: 200 })` will repeat forever, with a delay of 200ms between repetitions.\n * - `repeat({ count: 2, delay: 400 })` will repeat twice, with a delay of 400ms between repetitions.\n * - `repeat({ delay: (count) => timer(count * 1000) })` will repeat forever, but will have a delay that grows by one second for each repetition.\n *\n * ## Example\n *\n * Repeat a message stream\n *\n * ```ts\n * import { of, repeat } from 'rxjs';\n *\n * const source = of('Repeat message');\n * const result = source.pipe(repeat(3));\n *\n * result.subscribe(x => console.log(x));\n *\n * // Results\n * // 'Repeat message'\n * // 'Repeat message'\n * // 'Repeat message'\n * ```\n *\n * Repeat 3 values, 2 times\n *\n * ```ts\n * import { interval, take, repeat } from 'rxjs';\n *\n * const source = interval(1000);\n * const result = source.pipe(take(3), repeat(2));\n *\n * result.subscribe(x => console.log(x));\n *\n * // Results every second\n * // 0\n * // 1\n * // 2\n * // 0\n * // 1\n * // 2\n * ```\n *\n * Defining two complex repeats with delays on the same source.\n * Note that the second repeat cannot be called until the first\n * repeat as exhausted it's count.\n *\n * ```ts\n * import { defer, of, repeat } from 'rxjs';\n *\n * const source = defer(() => {\n *    return of(`Hello, it is ${new Date()}`)\n * });\n *\n * source.pipe(\n *    // Repeat 3 times with a delay of 1 second between repetitions\n *    repeat({\n *      count: 3,\n *      delay: 1000,\n *    }),\n *\n *    // *Then* repeat forever, but with an exponential step-back\n *    // maxing out at 1 minute.\n *    repeat({\n *      delay: (count) => timer(Math.min(60000, 2 ^ count * 1000))\n *    })\n * )\n * ```\n *\n * @see {@link repeatWhen}\n * @see {@link retry}\n *\n * @param count The number of times the source Observable items are repeated, a count of 0 will yield\n * an empty Observable.\n */\nexport function repeat<T>(countOrConfig?: number | RepeatConfig): MonoTypeOperatorFunction<T> {\n  let count = Infinity;\n  let delay: RepeatConfig['delay'];\n\n  if (countOrConfig != null) {\n    if (typeof countOrConfig === 'object') {\n      ({ count = Infinity, delay } = countOrConfig);\n    } else {\n      count = countOrConfig;\n    }\n  }\n\n  return count <= 0\n    ? () => EMPTY\n    : operate((source, subscriber) => {\n        let soFar = 0;\n        let sourceSub: Subscription | null;\n\n        const resubscribe = () => {\n          sourceSub?.unsubscribe();\n          sourceSub = null;\n          if (delay != null) {\n            const notifier = typeof delay === 'number' ? timer(delay) : innerFrom(delay(soFar));\n            const notifierSubscriber = createOperatorSubscriber(subscriber, () => {\n              notifierSubscriber.unsubscribe();\n              subscribeToSource();\n            });\n            notifier.subscribe(notifierSubscriber);\n          } else {\n            subscribeToSource();\n          }\n        };\n\n        const subscribeToSource = () => {\n          let syncUnsub = false;\n          sourceSub = source.subscribe(\n            createOperatorSubscriber(subscriber, undefined, () => {\n              if (++soFar < count) {\n                if (sourceSub) {\n                  resubscribe();\n                } else {\n                  syncUnsub = true;\n                }\n              } else {\n                subscriber.complete();\n              }\n            })\n          );\n\n          if (syncUnsub) {\n            resubscribe();\n          }\n        };\n\n        subscribeToSource();\n      });\n}\n", "import { OperatorFunction } from '../types';\nimport { operate } from '../util/lift';\nimport { scanInternals } from './scanInternals';\n\nexport function scan<V, A = V>(accumulator: (acc: A | V, value: V, index: number) => A): OperatorFunction<V, V | A>;\nexport function scan<V, A>(accumulator: (acc: A, value: V, index: number) => A, seed: A): OperatorFunction<V, A>;\nexport function scan<V, A, S>(accumulator: (acc: A | S, value: V, index: number) => A, seed: S): OperatorFunction<V, A>;\n\n// TODO: link to a \"redux pattern\" section in the guide (location TBD)\n\n/**\n * Useful for encapsulating and managing state. Applies an accumulator (or \"reducer function\")\n * to each value from the source after an initial state is established -- either via\n * a `seed` value (second argument), or from the first value from the source.\n *\n * <span class=\"informal\">It's like {@link reduce}, but emits the current\n * accumulation state after each update</span>\n *\n * ![](scan.png)\n *\n * This operator maintains an internal state and emits it after processing each value as follows:\n *\n * 1. First value arrives\n *   - If a `seed` value was supplied (as the second argument to `scan`), let `state = seed` and `value = firstValue`.\n *   - If NO `seed` value was supplied (no second argument), let `state = firstValue` and go to 3.\n * 2. Let `state = accumulator(state, value)`.\n *   - If an error is thrown by `accumulator`, notify the consumer of an error. The process ends.\n * 3. Emit `state`.\n * 4. Next value arrives, let `value = nextValue`, go to 2.\n *\n * ## Examples\n *\n * An average of previous numbers. This example shows how\n * not providing a `seed` can prime the stream with the\n * first value from the source.\n *\n * ```ts\n * import { of, scan, map } from 'rxjs';\n *\n * const numbers$ = of(1, 2, 3);\n *\n * numbers$\n *   .pipe(\n *     // Get the sum of the numbers coming in.\n *     scan((total, n) => total + n),\n *     // Get the average by dividing the sum by the total number\n *     // received so far (which is 1 more than the zero-based index).\n *     map((sum, index) => sum / (index + 1))\n *   )\n *   .subscribe(console.log);\n * ```\n *\n * The Fibonacci sequence. This example shows how you can use\n * a seed to prime accumulation process. Also... you know... Fibonacci.\n * So important to like, computers and stuff that its whiteboarded\n * in job interviews. Now you can show them the Rx version! (Please don't, haha)\n *\n * ```ts\n * import { interval, scan, map, startWith } from 'rxjs';\n *\n * const firstTwoFibs = [0, 1];\n * // An endless stream of Fibonacci numbers.\n * const fibonacci$ = interval(1000).pipe(\n *   // Scan to get the fibonacci numbers (after 0, 1)\n *   scan(([a, b]) => [b, a + b], firstTwoFibs),\n *   // Get the second number in the tuple, it's the one you calculated\n *   map(([, n]) => n),\n *   // Start with our first two digits :)\n *   startWith(...firstTwoFibs)\n * );\n *\n * fibonacci$.subscribe(console.log);\n * ```\n *\n * @see {@link expand}\n * @see {@link mergeScan}\n * @see {@link reduce}\n * @see {@link switchScan}\n *\n * @param accumulator A \"reducer function\". This will be called for each value after an initial state is\n * acquired.\n * @param seed The initial state. If this is not provided, the first value from the source will\n * be used as the initial state, and emitted without going through the accumulator. All subsequent values\n * will be processed by the accumulator function. If this is provided, all values will go through\n * the accumulator function.\n * @return A function that returns an Observable of the accumulated values.\n */\nexport function scan<V, A, S>(accumulator: (acc: V | A | S, value: V, index: number) => A, seed?: S): OperatorFunction<V, V | A> {\n  // providing a seed of `undefined` *should* be valid and trigger\n  // hasSeed! so don't use `seed !== undefined` checks!\n  // For this reason, we have to check it here at the original call site\n  // otherwise inside Operator/Subscriber we won't know if `undefined`\n  // means they didn't provide anything or if they literally provided `undefined`\n  return operate(scanInternals(accumulator, seed as S, arguments.length >= 2, true));\n}\n", "import { innerFrom } from '../observable/innerFrom';\nimport { Subject } from '../Subject';\nimport { SafeSubscriber } from '../Subscriber';\nimport { Subscription } from '../Subscription';\nimport { MonoTypeOperatorFunction, SubjectLike, ObservableInput } from '../types';\nimport { operate } from '../util/lift';\n\nexport interface ShareConfig<T> {\n  /**\n   * The factory used to create the subject that will connect the source observable to\n   * multicast consumers.\n   */\n  connector?: () => SubjectLike<T>;\n  /**\n   * If `true`, the resulting observable will reset internal state on error from source and return to a \"cold\" state. This\n   * allows the resulting observable to be \"retried\" in the event of an error.\n   * If `false`, when an error comes from the source it will push the error into the connecting subject, and the subject\n   * will remain the connecting subject, meaning the resulting observable will not go \"cold\" again, and subsequent retries\n   * or resubscriptions will resubscribe to that same subject. In all cases, RxJS subjects will emit the same error again, however\n   * {@link ReplaySubject} will also push its buffered values before pushing the error.\n   * It is also possible to pass a notifier factory returning an `ObservableInput` instead which grants more fine-grained\n   * control over how and when the reset should happen. This allows behaviors like conditional or delayed resets.\n   */\n  resetOnError?: boolean | ((error: any) => ObservableInput<any>);\n  /**\n   * If `true`, the resulting observable will reset internal state on completion from source and return to a \"cold\" state. This\n   * allows the resulting observable to be \"repeated\" after it is done.\n   * If `false`, when the source completes, it will push the completion through the connecting subject, and the subject\n   * will remain the connecting subject, meaning the resulting observable will not go \"cold\" again, and subsequent repeats\n   * or resubscriptions will resubscribe to that same subject.\n   * It is also possible to pass a notifier factory returning an `ObservableInput` instead which grants more fine-grained\n   * control over how and when the reset should happen. This allows behaviors like conditional or delayed resets.\n   */\n  resetOnComplete?: boolean | (() => ObservableInput<any>);\n  /**\n   * If `true`, when the number of subscribers to the resulting observable reaches zero due to those subscribers unsubscribing, the\n   * internal state will be reset and the resulting observable will return to a \"cold\" state. This means that the next\n   * time the resulting observable is subscribed to, a new subject will be created and the source will be subscribed to\n   * again.\n   * If `false`, when the number of subscribers to the resulting observable reaches zero due to unsubscription, the subject\n   * will remain connected to the source, and new subscriptions to the result will be connected through that same subject.\n   * It is also possible to pass a notifier factory returning an `ObservableInput` instead which grants more fine-grained\n   * control over how and when the reset should happen. This allows behaviors like conditional or delayed resets.\n   */\n  resetOnRefCountZero?: boolean | (() => ObservableInput<any>);\n}\n\nexport function share<T>(): MonoTypeOperatorFunction<T>;\n\nexport function share<T>(options: ShareConfig<T>): MonoTypeOperatorFunction<T>;\n\n/**\n * Returns a new Observable that multicasts (shares) the original Observable. As long as there is at least one\n * Subscriber this Observable will be subscribed and emitting data. When all subscribers have unsubscribed it will\n * unsubscribe from the source Observable. Because the Observable is multicasting it makes the stream `hot`.\n * This is an alias for `multicast(() => new Subject()), refCount()`.\n *\n * The subscription to the underlying source Observable can be reset (unsubscribe and resubscribe for new subscribers),\n * if the subscriber count to the shared observable drops to 0, or if the source Observable errors or completes. It is\n * possible to use notifier factories for the resets to allow for behaviors like conditional or delayed resets. Please\n * note that resetting on error or complete of the source Observable does not behave like a transparent retry or restart\n * of the source because the error or complete will be forwarded to all subscribers and their subscription will be\n * closed. Only new subscribers after a reset on error or complete happened will cause a fresh subscription to the\n * source. To achieve transparent retries or restarts pipe the source through appropriate operators before sharing.\n *\n * ![](share.png)\n *\n * ## Example\n *\n * Generate new multicast Observable from the `source` Observable value\n *\n * ```ts\n * import { interval, tap, map, take, share } from 'rxjs';\n *\n * const source = interval(1000).pipe(\n *   tap(x => console.log('Processing: ', x)),\n *   map(x => x * x),\n *   take(6),\n *   share()\n * );\n *\n * source.subscribe(x => console.log('subscription 1: ', x));\n * source.subscribe(x => console.log('subscription 2: ', x));\n *\n * // Logs:\n * // Processing: 0\n * // subscription 1: 0\n * // subscription 2: 0\n * // Processing: 1\n * // subscription 1: 1\n * // subscription 2: 1\n * // Processing: 2\n * // subscription 1: 4\n * // subscription 2: 4\n * // Processing: 3\n * // subscription 1: 9\n * // subscription 2: 9\n * // Processing: 4\n * // subscription 1: 16\n * // subscription 2: 16\n * // Processing: 5\n * // subscription 1: 25\n * // subscription 2: 25\n * ```\n *\n * ## Example with notifier factory: Delayed reset\n *\n * ```ts\n * import { interval, take, share, timer } from 'rxjs';\n *\n * const source = interval(1000).pipe(\n *   take(3),\n *   share({\n *     resetOnRefCountZero: () => timer(1000)\n *   })\n * );\n *\n * const subscriptionOne = source.subscribe(x => console.log('subscription 1: ', x));\n * setTimeout(() => subscriptionOne.unsubscribe(), 1300);\n *\n * setTimeout(() => source.subscribe(x => console.log('subscription 2: ', x)), 1700);\n *\n * setTimeout(() => source.subscribe(x => console.log('subscription 3: ', x)), 5000);\n *\n * // Logs:\n * // subscription 1:  0\n * // (subscription 1 unsubscribes here)\n * // (subscription 2 subscribes here ~400ms later, source was not reset)\n * // subscription 2:  1\n * // subscription 2:  2\n * // (subscription 2 unsubscribes here)\n * // (subscription 3 subscribes here ~2000ms later, source did reset before)\n * // subscription 3:  0\n * // subscription 3:  1\n * // subscription 3:  2\n * ```\n *\n * @see {@link shareReplay}\n *\n * @return A function that returns an Observable that mirrors the source.\n */\nexport function share<T>(options: ShareConfig<T> = {}): MonoTypeOperatorFunction<T> {\n  const { connector = () => new Subject<T>(), resetOnError = true, resetOnComplete = true, resetOnRefCountZero = true } = options;\n  // It's necessary to use a wrapper here, as the _operator_ must be\n  // referentially transparent. Otherwise, it cannot be used in calls to the\n  // static `pipe` function - to create a partial pipeline.\n  //\n  // The _operator function_ - the function returned by the _operator_ - will\n  // not be referentially transparent - as it shares its source - but the\n  // _operator function_ is called when the complete pipeline is composed via a\n  // call to a source observable's `pipe` method - not when the static `pipe`\n  // function is called.\n  return (wrapperSource) => {\n    let connection: SafeSubscriber<T> | undefined;\n    let resetConnection: Subscription | undefined;\n    let subject: SubjectLike<T> | undefined;\n    let refCount = 0;\n    let hasCompleted = false;\n    let hasErrored = false;\n\n    const cancelReset = () => {\n      resetConnection?.unsubscribe();\n      resetConnection = undefined;\n    };\n    // Used to reset the internal state to a \"cold\"\n    // state, as though it had never been subscribed to.\n    const reset = () => {\n      cancelReset();\n      connection = subject = undefined;\n      hasCompleted = hasErrored = false;\n    };\n    const resetAndUnsubscribe = () => {\n      // We need to capture the connection before\n      // we reset (if we need to reset).\n      const conn = connection;\n      reset();\n      conn?.unsubscribe();\n    };\n\n    return operate<T, T>((source, subscriber) => {\n      refCount++;\n      if (!hasErrored && !hasCompleted) {\n        cancelReset();\n      }\n\n      // Create the subject if we don't have one yet. Grab a local reference to\n      // it as well, which avoids non-null assertions when using it and, if we\n      // connect to it now, then error/complete need a reference after it was\n      // reset.\n      const dest = (subject = subject ?? connector());\n\n      // Add the finalization directly to the subscriber - instead of returning it -\n      // so that the handling of the subscriber's unsubscription will be wired\n      // up _before_ the subscription to the source occurs. This is done so that\n      // the assignment to the source connection's `closed` property will be seen\n      // by synchronous firehose sources.\n      subscriber.add(() => {\n        refCount--;\n\n        // If we're resetting on refCount === 0, and it's 0, we only want to do\n        // that on \"unsubscribe\", really. Resetting on error or completion is a different\n        // configuration.\n        if (refCount === 0 && !hasErrored && !hasCompleted) {\n          resetConnection = handleReset(resetAndUnsubscribe, resetOnRefCountZero);\n        }\n      });\n\n      // The following line adds the subscription to the subscriber passed.\n      // Basically, `subscriber === dest.subscribe(subscriber)` is `true`.\n      dest.subscribe(subscriber);\n\n      if (\n        !connection &&\n        // Check this shareReplay is still activate - it can be reset to 0\n        // and be \"unsubscribed\" _before_ it actually subscribes.\n        // If we were to subscribe then, it'd leak and get stuck.\n        refCount > 0\n      ) {\n        // We need to create a subscriber here - rather than pass an observer and\n        // assign the returned subscription to connection - because it's possible\n        // for reentrant subscriptions to the shared observable to occur and in\n        // those situations we want connection to be already-assigned so that we\n        // don't create another connection to the source.\n        connection = new SafeSubscriber({\n          next: (value) => dest.next(value),\n          error: (err) => {\n            hasErrored = true;\n            cancelReset();\n            resetConnection = handleReset(reset, resetOnError, err);\n            dest.error(err);\n          },\n          complete: () => {\n            hasCompleted = true;\n            cancelReset();\n            resetConnection = handleReset(reset, resetOnComplete);\n            dest.complete();\n          },\n        });\n        innerFrom(source).subscribe(connection);\n      }\n    })(wrapperSource);\n  };\n}\n\nfunction handleReset<T extends unknown[] = never[]>(\n  reset: () => void,\n  on: boolean | ((...args: T) => ObservableInput<any>),\n  ...args: T\n): Subscription | undefined {\n  if (on === true) {\n    reset();\n    return;\n  }\n\n  if (on === false) {\n    return;\n  }\n\n  const onSubscriber = new SafeSubscriber({\n    next: () => {\n      onSubscriber.unsubscribe();\n      reset();\n    },\n  });\n\n  return innerFrom(on(...args)).subscribe(onSubscriber);\n}\n", "import { ReplaySubject } from '../ReplaySubject';\nimport { MonoTypeOperatorFunction, SchedulerLike } from '../types';\nimport { share } from './share';\n\nexport interface ShareReplayConfig {\n  bufferSize?: number;\n  windowTime?: number;\n  refCount: boolean;\n  scheduler?: SchedulerLike;\n}\n\nexport function shareReplay<T>(config: ShareReplayConfig): MonoTypeOperatorFunction<T>;\nexport function shareReplay<T>(bufferSize?: number, windowTime?: number, scheduler?: SchedulerLike): MonoTypeOperatorFunction<T>;\n\n/**\n * Share source and replay specified number of emissions on subscription.\n *\n * This operator is a specialization of `replay` that connects to a source observable\n * and multicasts through a `ReplaySubject` constructed with the specified arguments.\n * A successfully completed source will stay cached in the `shareReplay`ed observable forever,\n * but an errored source can be retried.\n *\n * ## Why use `shareReplay`?\n *\n * You generally want to use `shareReplay` when you have side-effects or taxing computations\n * that you do not wish to be executed amongst multiple subscribers.\n * It may also be valuable in situations where you know you will have late subscribers to\n * a stream that need access to previously emitted values.\n * This ability to replay values on subscription is what differentiates {@link share} and `shareReplay`.\n *\n * ## Reference counting\n *\n * By default `shareReplay` will use `refCount` of false, meaning that it will _not_ unsubscribe the\n * source when the reference counter drops to zero, i.e. the inner `ReplaySubject` will _not_ be unsubscribed\n * (and potentially run for ever).\n * This is the default as it is expected that `shareReplay` is often used to keep around expensive to setup\n * observables which we want to keep running instead of having to do the expensive setup again.\n *\n * As of RXJS version 6.4.0 a new overload signature was added to allow for manual control over what\n * happens when the operators internal reference counter drops to zero.\n * If `refCount` is true, the source will be unsubscribed from once the reference count drops to zero, i.e.\n * the inner `ReplaySubject` will be unsubscribed. All new subscribers will receive value emissions from a\n * new `ReplaySubject` which in turn will cause a new subscription to the source observable.\n *\n * ## Examples\n *\n * Example with a third subscriber coming late to the party\n *\n * ```ts\n * import { interval, take, shareReplay } from 'rxjs';\n *\n * const shared$ = interval(2000).pipe(\n *   take(6),\n *   shareReplay(3)\n * );\n *\n * shared$.subscribe(x => console.log('sub A: ', x));\n * shared$.subscribe(y => console.log('sub B: ', y));\n *\n * setTimeout(() => {\n *   shared$.subscribe(y => console.log('sub C: ', y));\n * }, 11000);\n *\n * // Logs:\n * // (after ~2000 ms)\n * // sub A: 0\n * // sub B: 0\n * // (after ~4000 ms)\n * // sub A: 1\n * // sub B: 1\n * // (after ~6000 ms)\n * // sub A: 2\n * // sub B: 2\n * // (after ~8000 ms)\n * // sub A: 3\n * // sub B: 3\n * // (after ~10000 ms)\n * // sub A: 4\n * // sub B: 4\n * // (after ~11000 ms, sub C gets the last 3 values)\n * // sub C: 2\n * // sub C: 3\n * // sub C: 4\n * // (after ~12000 ms)\n * // sub A: 5\n * // sub B: 5\n * // sub C: 5\n * ```\n *\n * Example for `refCount` usage\n *\n * ```ts\n * import { Observable, tap, interval, shareReplay, take } from 'rxjs';\n *\n * const log = <T>(name: string, source: Observable<T>) => source.pipe(\n *   tap({\n *     subscribe: () => console.log(`${ name }: subscribed`),\n *     next: value => console.log(`${ name }: ${ value }`),\n *     complete: () => console.log(`${ name }: completed`),\n *     finalize: () => console.log(`${ name }: unsubscribed`)\n *   })\n * );\n *\n * const obs$ = log('source', interval(1000));\n *\n * const shared$ = log('shared', obs$.pipe(\n *   shareReplay({ bufferSize: 1, refCount: true }),\n *   take(2)\n * ));\n *\n * shared$.subscribe(x => console.log('sub A: ', x));\n * shared$.subscribe(y => console.log('sub B: ', y));\n *\n * // PRINTS:\n * // shared: subscribed <-- reference count = 1\n * // source: subscribed\n * // shared: subscribed <-- reference count = 2\n * // source: 0\n * // shared: 0\n * // sub A: 0\n * // shared: 0\n * // sub B: 0\n * // source: 1\n * // shared: 1\n * // sub A: 1\n * // shared: completed <-- take(2) completes the subscription for sub A\n * // shared: unsubscribed <-- reference count = 1\n * // shared: 1\n * // sub B: 1\n * // shared: completed <-- take(2) completes the subscription for sub B\n * // shared: unsubscribed <-- reference count = 0\n * // source: unsubscribed <-- replaySubject unsubscribes from source observable because the reference count dropped to 0 and refCount is true\n *\n * // In case of refCount being false, the unsubscribe is never called on the source and the source would keep on emitting, even if no subscribers\n * // are listening.\n * // source: 2\n * // source: 3\n * // source: 4\n * // ...\n * ```\n *\n * @see {@link publish}\n * @see {@link share}\n * @see {@link publishReplay}\n *\n * @param configOrBufferSize Maximum element count of the replay buffer or {@link ShareReplayConfig configuration}\n * object.\n * @param windowTime Maximum time length of the replay buffer in milliseconds.\n * @param scheduler Scheduler where connected observers within the selector function\n * will be invoked on.\n * @return A function that returns an Observable sequence that contains the\n * elements of a sequence produced by multicasting the source sequence within a\n * selector function.\n */\nexport function shareReplay<T>(\n  configOrBufferSize?: ShareReplayConfig | number,\n  windowTime?: number,\n  scheduler?: SchedulerLike\n): MonoTypeOperatorFunction<T> {\n  let bufferSize: number;\n  let refCount = false;\n  if (configOrBufferSize && typeof configOrBufferSize === 'object') {\n    ({ bufferSize = Infinity, windowTime = Infinity, refCount = false, scheduler } = configOrBufferSize);\n  } else {\n    bufferSize = (configOrBufferSize ?? Infinity) as number;\n  }\n  return share<T>({\n    connector: () => new ReplaySubject(bufferSize, windowTime, scheduler),\n    resetOnError: true,\n    resetOnComplete: false,\n    resetOnRefCountZero: refCount,\n  });\n}\n", "import { MonoTypeOperatorFunction } from '../types';\nimport { filter } from './filter';\n\n/**\n * Returns an Observable that skips the first `count` items emitted by the source Observable.\n *\n * ![](skip.png)\n *\n * Skips the values until the sent notifications are equal or less than provided skip count. It raises\n * an error if skip count is equal or more than the actual number of emits and source raises an error.\n *\n * ## Example\n *\n * Skip the values before the emission\n *\n * ```ts\n * import { interval, skip } from 'rxjs';\n *\n * // emit every half second\n * const source = interval(500);\n * // skip the first 10 emitted values\n * const result = source.pipe(skip(10));\n *\n * result.subscribe(value => console.log(value));\n * // output: 10...11...12...13...\n * ```\n *\n * @see {@link last}\n * @see {@link skipWhile}\n * @see {@link skipUntil}\n * @see {@link skipLast}\n *\n * @param {Number} count - The number of times, items emitted by source Observable should be skipped.\n * @return A function that returns an Observable that skips the first `count`\n * values emitted by the source Observable.\n */\nexport function skip<T>(count: number): MonoTypeOperatorFunction<T> {\n  return filter((_, index) => count <= index);\n}\n", "import { MonoTypeOperatorFunction, ObservableInput } from '../types';\nimport { operate } from '../util/lift';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\nimport { innerFrom } from '../observable/innerFrom';\nimport { noop } from '../util/noop';\n\n/**\n * Returns an Observable that skips items emitted by the source Observable until a second Observable emits an item.\n *\n * The `skipUntil` operator causes the observable stream to skip the emission of values until the passed in observable\n * emits the first value. This can be particularly useful in combination with user interactions, responses of HTTP\n * requests or waiting for specific times to pass by.\n *\n * ![](skipUntil.png)\n *\n * Internally, the `skipUntil` operator subscribes to the passed in `notifier` `ObservableInput` (which gets converted\n * to an Observable) in order to recognize the emission of its first value. When `notifier` emits next, the operator\n * unsubscribes from it and starts emitting the values of the *source* observable until it completes or errors. It\n * will never let the *source* observable emit any values if the `notifier` completes or throws an error without\n * emitting a value before.\n *\n * ## Example\n *\n * In the following example, all emitted values of the interval observable are skipped until the user clicks anywhere\n * within the page\n *\n * ```ts\n * import { interval, fromEvent, skipUntil } from 'rxjs';\n *\n * const intervalObservable = interval(1000);\n * const click = fromEvent(document, 'click');\n *\n * const emitAfterClick = intervalObservable.pipe(\n *   skipUntil(click)\n * );\n * // clicked at 4.6s. output: 5...6...7...8........ or\n * // clicked at 7.3s. output: 8...9...10..11.......\n * emitAfterClick.subscribe(value => console.log(value));\n * ```\n *\n * @see {@link last}\n * @see {@link skip}\n * @see {@link skipWhile}\n * @see {@link skipLast}\n *\n * @param notifier An `ObservableInput` that has to emit an item before the source Observable elements begin to\n * be mirrored by the resulting Observable.\n * @return A function that returns an Observable that skips items from the\n * source Observable until the `notifier` Observable emits an item, then emits the\n * remaining items.\n */\nexport function skipUntil<T>(notifier: ObservableInput<any>): MonoTypeOperatorFunction<T> {\n  return operate((source, subscriber) => {\n    let taking = false;\n\n    const skipSubscriber = createOperatorSubscriber(\n      subscriber,\n      () => {\n        skipSubscriber?.unsubscribe();\n        taking = true;\n      },\n      noop\n    );\n\n    innerFrom(notifier).subscribe(skipSubscriber);\n\n    source.subscribe(createOperatorSubscriber(subscriber, (value) => taking && subscriber.next(value)));\n  });\n}\n", "import { concat } from '../observable/concat';\nimport { OperatorFunction, SchedulerLike, ValueFromArray } from '../types';\nimport { popScheduler } from '../util/args';\nimport { operate } from '../util/lift';\n\n// Devs are more likely to pass null or undefined than they are a scheduler\n// without accompanying values. To make things easier for (naughty) devs who\n// use the `strictNullChecks: false` TypeScript compiler option, these\n// overloads with explicit null and undefined values are included.\n\nexport function startWith<T>(value: null): OperatorFunction<T, T | null>;\nexport function startWith<T>(value: undefined): OperatorFunction<T, T | undefined>;\n\n/** @deprecated The `scheduler` parameter will be removed in v8. Use `scheduled` and `concatAll`. Details: https://rxjs.dev/deprecations/scheduler-argument */\nexport function startWith<T, A extends readonly unknown[] = T[]>(\n  ...valuesAndScheduler: [...A, SchedulerLike]\n): OperatorFunction<T, T | ValueFromArray<A>>;\nexport function startWith<T, A extends readonly unknown[] = T[]>(...values: A): OperatorFunction<T, T | ValueFromArray<A>>;\n\n/**\n * Returns an observable that, at the moment of subscription, will synchronously emit all\n * values provided to this operator, then subscribe to the source and mirror all of its emissions\n * to subscribers.\n *\n * This is a useful way to know when subscription has occurred on an existing observable.\n *\n * <span class=\"informal\">First emits its arguments in order, and then any\n * emissions from the source.</span>\n *\n * ![](startWith.png)\n *\n * ## Examples\n *\n * Emit a value when a timer starts.\n *\n * ```ts\n * import { timer, map, startWith } from 'rxjs';\n *\n * timer(1000)\n *   .pipe(\n *     map(() => 'timer emit'),\n *     startWith('timer start')\n *   )\n *   .subscribe(x => console.log(x));\n *\n * // results:\n * // 'timer start'\n * // 'timer emit'\n * ```\n *\n * @param values Items you want the modified Observable to emit first.\n * @return A function that returns an Observable that synchronously emits\n * provided values before subscribing to the source Observable.\n *\n * @see {@link endWith}\n * @see {@link finalize}\n * @see {@link concat}\n */\nexport function startWith<T, D>(...values: D[]): OperatorFunction<T, T | D> {\n  const scheduler = popScheduler(values);\n  return operate((source, subscriber) => {\n    // Here we can't pass `undefined` as a scheduler, because if we did, the\n    // code inside of `concat` would be confused by the `undefined`, and treat it\n    // like an invalid observable. So we have to split it two different ways.\n    (scheduler ? concat(values, source, scheduler) : concat(values, source)).subscribe(subscriber);\n  });\n}\n", "import { Subscriber } from '../Subscriber';\nimport { ObservableInput, OperatorFunction, ObservedValueOf } from '../types';\nimport { innerFrom } from '../observable/innerFrom';\nimport { operate } from '../util/lift';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\n\n/* tslint:disable:max-line-length */\nexport function switchMap<T, O extends ObservableInput<any>>(\n  project: (value: T, index: number) => O\n): OperatorFunction<T, ObservedValueOf<O>>;\n/** @deprecated The `resultSelector` parameter will be removed in v8. Use an inner `map` instead. Details: https://rxjs.dev/deprecations/resultSelector */\nexport function switchMap<T, O extends ObservableInput<any>>(\n  project: (value: T, index: number) => O,\n  resultSelector: undefined\n): OperatorFunction<T, ObservedValueOf<O>>;\n/** @deprecated The `resultSelector` parameter will be removed in v8. Use an inner `map` instead. Details: https://rxjs.dev/deprecations/resultSelector */\nexport function switchMap<T, R, O extends ObservableInput<any>>(\n  project: (value: T, index: number) => O,\n  resultSelector: (outerValue: T, innerValue: ObservedValueOf<O>, outerIndex: number, innerIndex: number) => R\n): OperatorFunction<T, R>;\n/* tslint:enable:max-line-length */\n\n/**\n * Projects each source value to an Observable which is merged in the output\n * Observable, emitting values only from the most recently projected Observable.\n *\n * <span class=\"informal\">Maps each value to an Observable, then flattens all of\n * these inner Observables using {@link switchAll}.</span>\n *\n * ![](switchMap.png)\n *\n * Returns an Observable that emits items based on applying a function that you\n * supply to each item emitted by the source Observable, where that function\n * returns an (so-called \"inner\") Observable. Each time it observes one of these\n * inner Observables, the output Observable begins emitting the items emitted by\n * that inner Observable. When a new inner Observable is emitted, `switchMap`\n * stops emitting items from the earlier-emitted inner Observable and begins\n * emitting items from the new one. It continues to behave like this for\n * subsequent inner Observables.\n *\n * ## Example\n *\n * Generate new Observable according to source Observable values\n *\n * ```ts\n * import { of, switchMap } from 'rxjs';\n *\n * const switched = of(1, 2, 3).pipe(switchMap(x => of(x, x ** 2, x ** 3)));\n * switched.subscribe(x => console.log(x));\n * // outputs\n * // 1\n * // 1\n * // 1\n * // 2\n * // 4\n * // 8\n * // 3\n * // 9\n * // 27\n * ```\n *\n * Restart an interval Observable on every click event\n *\n * ```ts\n * import { fromEvent, switchMap, interval } from 'rxjs';\n *\n * const clicks = fromEvent(document, 'click');\n * const result = clicks.pipe(switchMap(() => interval(1000)));\n * result.subscribe(x => console.log(x));\n * ```\n *\n * @see {@link concatMap}\n * @see {@link exhaustMap}\n * @see {@link mergeMap}\n * @see {@link switchAll}\n * @see {@link switchMapTo}\n *\n * @param {function(value: T, index: number): ObservableInput} project A function\n * that, when applied to an item emitted by the source Observable, returns an\n * Observable.\n * @return A function that returns an Observable that emits the result of\n * applying the projection function (and the optional deprecated\n * `resultSelector`) to each item emitted by the source Observable and taking\n * only the values from the most recently projected inner Observable.\n */\nexport function switchMap<T, R, O extends ObservableInput<any>>(\n  project: (value: T, index: number) => O,\n  resultSelector?: (outerValue: T, innerValue: ObservedValueOf<O>, outerIndex: number, innerIndex: number) => R\n): OperatorFunction<T, ObservedValueOf<O> | R> {\n  return operate((source, subscriber) => {\n    let innerSubscriber: Subscriber<ObservedValueOf<O>> | null = null;\n    let index = 0;\n    // Whether or not the source subscription has completed\n    let isComplete = false;\n\n    // We only complete the result if the source is complete AND we don't have an active inner subscription.\n    // This is called both when the source completes and when the inners complete.\n    const checkComplete = () => isComplete && !innerSubscriber && subscriber.complete();\n\n    source.subscribe(\n      createOperatorSubscriber(\n        subscriber,\n        (value) => {\n          // Cancel the previous inner subscription if there was one\n          innerSubscriber?.unsubscribe();\n          let innerIndex = 0;\n          const outerIndex = index++;\n          // Start the next inner subscription\n          innerFrom(project(value, outerIndex)).subscribe(\n            (innerSubscriber = createOperatorSubscriber(\n              subscriber,\n              // When we get a new inner value, next it through. Note that this is\n              // handling the deprecate result selector here. This is because with this architecture\n              // it ends up being smaller than using the map operator.\n              (innerValue) => subscriber.next(resultSelector ? resultSelector(value, innerValue, outerIndex, innerIndex++) : innerValue),\n              () => {\n                // The inner has completed. Null out the inner subscriber to\n                // free up memory and to signal that we have no inner subscription\n                // currently.\n                innerSubscriber = null!;\n                checkComplete();\n              }\n            ))\n          );\n        },\n        () => {\n          isComplete = true;\n          checkComplete();\n        }\n      )\n    );\n  });\n}\n", "import { MonoTypeOperatorFunction, ObservableInput } from '../types';\nimport { operate } from '../util/lift';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\nimport { innerFrom } from '../observable/innerFrom';\nimport { noop } from '../util/noop';\n\n/**\n * Emits the values emitted by the source Observable until a `notifier`\n * Observable emits a value.\n *\n * <span class=\"informal\">Lets values pass until a second Observable,\n * `notifier`, emits a value. Then, it completes.</span>\n *\n * ![](takeUntil.png)\n *\n * `takeUntil` subscribes and begins mirroring the source Observable. It also\n * monitors a second Observable, `notifier` that you provide. If the `notifier`\n * emits a value, the output Observable stops mirroring the source Observable\n * and completes. If the `notifier` doesn't emit any value and completes\n * then `takeUntil` will pass all values.\n *\n * ## Example\n *\n * Tick every second until the first click happens\n *\n * ```ts\n * import { interval, fromEvent, takeUntil } from 'rxjs';\n *\n * const source = interval(1000);\n * const clicks = fromEvent(document, 'click');\n * const result = source.pipe(takeUntil(clicks));\n * result.subscribe(x => console.log(x));\n * ```\n *\n * @see {@link take}\n * @see {@link takeLast}\n * @see {@link takeWhile}\n * @see {@link skip}\n *\n * @param {Observable} notifier The Observable whose first emitted value will\n * cause the output Observable of `takeUntil` to stop emitting values from the\n * source Observable.\n * @return A function that returns an Observable that emits the values from the\n * source Observable until `notifier` emits its first value.\n */\nexport function takeUntil<T>(notifier: ObservableInput<any>): MonoTypeOperatorFunction<T> {\n  return operate((source, subscriber) => {\n    innerFrom(notifier).subscribe(createOperatorSubscriber(subscriber, () => subscriber.complete(), noop));\n    !subscriber.closed && source.subscribe(subscriber);\n  });\n}\n", "import { OperatorFunction, MonoTypeOperatorFunction, TruthyTypesOf } from '../types';\nimport { operate } from '../util/lift';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\n\nexport function takeWhile<T>(predicate: BooleanConstructor, inclusive: true): MonoTypeOperatorFunction<T>;\nexport function takeWhile<T>(predicate: BooleanConstructor, inclusive: false): OperatorFunction<T, TruthyTypesOf<T>>;\nexport function takeWhile<T>(predicate: BooleanConstructor): OperatorFunction<T, TruthyTypesOf<T>>;\nexport function takeWhile<T, S extends T>(predicate: (value: T, index: number) => value is S): OperatorFunction<T, S>;\nexport function takeWhile<T, S extends T>(predicate: (value: T, index: number) => value is S, inclusive: false): OperatorFunction<T, S>;\nexport function takeWhile<T>(predicate: (value: T, index: number) => boolean, inclusive?: boolean): MonoTypeOperatorFunction<T>;\n\n/**\n * Emits values emitted by the source Observable so long as each value satisfies\n * the given `predicate`, and then completes as soon as this `predicate` is not\n * satisfied.\n *\n * <span class=\"informal\">Takes values from the source only while they pass the\n * condition given. When the first value does not satisfy, it completes.</span>\n *\n * ![](takeWhile.png)\n *\n * `takeWhile` subscribes and begins mirroring the source Observable. Each value\n * emitted on the source is given to the `predicate` function which returns a\n * boolean, representing a condition to be satisfied by the source values. The\n * output Observable emits the source values until such time as the `predicate`\n * returns false, at which point `takeWhile` stops mirroring the source\n * Observable and completes the output Observable.\n *\n * ## Example\n *\n * Emit click events only while the clientX property is greater than 200\n *\n * ```ts\n * import { fromEvent, takeWhile } from 'rxjs';\n *\n * const clicks = fromEvent<PointerEvent>(document, 'click');\n * const result = clicks.pipe(takeWhile(ev => ev.clientX > 200));\n * result.subscribe(x => console.log(x));\n * ```\n *\n * @see {@link take}\n * @see {@link takeLast}\n * @see {@link takeUntil}\n * @see {@link skip}\n *\n * @param {function(value: T, index: number): boolean} predicate A function that\n * evaluates a value emitted by the source Observable and returns a boolean.\n * Also takes the (zero-based) index as the second argument.\n * @param {boolean} inclusive When set to `true` the value that caused\n * `predicate` to return `false` will also be emitted.\n * @return A function that returns an Observable that emits values from the\n * source Observable so long as each value satisfies the condition defined by\n * the `predicate`, then completes.\n */\nexport function takeWhile<T>(predicate: (value: T, index: number) => boolean, inclusive = false): MonoTypeOperatorFunction<T> {\n  return operate((source, subscriber) => {\n    let index = 0;\n    source.subscribe(\n      createOperatorSubscriber(subscriber, (value) => {\n        const result = predicate(value, index++);\n        (result || inclusive) && subscriber.next(value);\n        !result && subscriber.complete();\n      })\n    );\n  });\n}\n", "import { MonoTypeOperatorFunction, Observer } from '../types';\nimport { isFunction } from '../util/isFunction';\nimport { operate } from '../util/lift';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\nimport { identity } from '../util/identity';\n\n/**\n * An extension to the {@link Observer} interface used only by the {@link tap} operator.\n *\n * It provides a useful set of callbacks a user can register to do side-effects in\n * cases other than what the usual {@link Observer} callbacks are\n * ({@link guide/glossary-and-semantics#next next},\n * {@link guide/glossary-and-semantics#error error} and/or\n * {@link guide/glossary-and-semantics#complete complete}).\n *\n * ## Example\n *\n * ```ts\n * import { fromEvent, switchMap, tap, interval, take } from 'rxjs';\n *\n * const source$ = fromEvent(document, 'click');\n * const result$ = source$.pipe(\n *   switchMap((_, i) => i % 2 === 0\n *     ? fromEvent(document, 'mousemove').pipe(\n *         tap({\n *           subscribe: () => console.log('Subscribed to the mouse move events after click #' + i),\n *           unsubscribe: () => console.log('Mouse move events #' + i + ' unsubscribed'),\n *           finalize: () => console.log('Mouse move events #' + i + ' finalized')\n *         })\n *       )\n *     : interval(1_000).pipe(\n *         take(5),\n *         tap({\n *           subscribe: () => console.log('Subscribed to the 1-second interval events after click #' + i),\n *           unsubscribe: () => console.log('1-second interval events #' + i + ' unsubscribed'),\n *           finalize: () => console.log('1-second interval events #' + i + ' finalized')\n *         })\n *       )\n *   )\n * );\n *\n * const subscription = result$.subscribe({\n *   next: console.log\n * });\n *\n * setTimeout(() => {\n *   console.log('Unsubscribe after 60 seconds');\n *   subscription.unsubscribe();\n * }, 60_000);\n * ```\n */\nexport interface TapObserver<T> extends Observer<T> {\n  /**\n   * The callback that `tap` operator invokes at the moment when the source Observable\n   * gets subscribed to.\n   */\n  subscribe: () => void;\n  /**\n   * The callback that `tap` operator invokes when an explicit\n   * {@link guide/glossary-and-semantics#unsubscription unsubscribe} happens. It won't get invoked on\n   * `error` or `complete` events.\n   */\n  unsubscribe: () => void;\n  /**\n   * The callback that `tap` operator invokes when any kind of\n   * {@link guide/glossary-and-semantics#finalization finalization} happens - either when\n   * the source Observable `error`s or `complete`s or when it gets explicitly unsubscribed\n   * by the user. There is no difference in using this callback or the {@link finalize}\n   * operator, but if you're already using `tap` operator, you can use this callback\n   * instead. You'd get the same result in either case.\n   */\n  finalize: () => void;\n}\nexport function tap<T>(observerOrNext?: Partial<TapObserver<T>> | ((value: T) => void)): MonoTypeOperatorFunction<T>;\n/** @deprecated Instead of passing separate callback arguments, use an observer argument. Signatures taking separate callback arguments will be removed in v8. Details: https://rxjs.dev/deprecations/subscribe-arguments */\nexport function tap<T>(\n  next?: ((value: T) => void) | null,\n  error?: ((error: any) => void) | null,\n  complete?: (() => void) | null\n): MonoTypeOperatorFunction<T>;\n\n/**\n * Used to perform side-effects for notifications from the source observable\n *\n * <span class=\"informal\">Used when you want to affect outside state with a notification without altering the notification</span>\n *\n * ![](tap.png)\n *\n * Tap is designed to allow the developer a designated place to perform side effects. While you _could_ perform side-effects\n * inside of a `map` or a `mergeMap`, that would make their mapping functions impure, which isn't always a big deal, but will\n * make it so you can't do things like memoize those functions. The `tap` operator is designed solely for such side-effects to\n * help you remove side-effects from other operations.\n *\n * For any notification, next, error, or complete, `tap` will call the appropriate callback you have provided to it, via a function\n * reference, or a partial observer, then pass that notification down the stream.\n *\n * The observable returned by `tap` is an exact mirror of the source, with one exception: Any error that occurs -- synchronously -- in a handler\n * provided to `tap` will be emitted as an error from the returned observable.\n *\n * > Be careful! You can mutate objects as they pass through the `tap` operator's handlers.\n *\n * The most common use of `tap` is actually for debugging. You can place a `tap(console.log)` anywhere\n * in your observable `pipe`, log out the notifications as they are emitted by the source returned by the previous\n * operation.\n *\n * ## Examples\n *\n * Check a random number before it is handled. Below is an observable that will use a random number between 0 and 1,\n * and emit `'big'` or `'small'` depending on the size of that number. But we wanted to log what the original number\n * was, so we have added a `tap(console.log)`.\n *\n * ```ts\n * import { of, tap, map } from 'rxjs';\n *\n * of(Math.random()).pipe(\n *   tap(console.log),\n *   map(n => n > 0.5 ? 'big' : 'small')\n * ).subscribe(console.log);\n * ```\n *\n * Using `tap` to analyze a value and force an error. Below is an observable where in our system we only\n * want to emit numbers 3 or less we get from another source. We can force our observable to error\n * using `tap`.\n *\n * ```ts\n * import { of, tap } from 'rxjs';\n *\n * const source = of(1, 2, 3, 4, 5);\n *\n * source.pipe(\n *   tap(n => {\n *     if (n > 3) {\n *       throw new TypeError(`Value ${ n } is greater than 3`);\n *     }\n *   })\n * )\n * .subscribe({ next: console.log, error: err => console.log(err.message) });\n * ```\n *\n * We want to know when an observable completes before moving on to the next observable. The system\n * below will emit a random series of `'X'` characters from 3 different observables in sequence. The\n * only way we know when one observable completes and moves to the next one, in this case, is because\n * we have added a `tap` with the side effect of logging to console.\n *\n * ```ts\n * import { of, concatMap, interval, take, map, tap } from 'rxjs';\n *\n * of(1, 2, 3).pipe(\n *   concatMap(n => interval(1000).pipe(\n *     take(Math.round(Math.random() * 10)),\n *     map(() => 'X'),\n *     tap({ complete: () => console.log(`Done with ${ n }`) })\n *   ))\n * )\n * .subscribe(console.log);\n * ```\n *\n * @see {@link finalize}\n * @see {@link TapObserver}\n *\n * @param observerOrNext A next handler or partial observer\n * @param error An error handler\n * @param complete A completion handler\n * @return A function that returns an Observable identical to the source, but\n * runs the specified Observer or callback(s) for each item.\n */\nexport function tap<T>(\n  observerOrNext?: Partial<TapObserver<T>> | ((value: T) => void) | null,\n  error?: ((e: any) => void) | null,\n  complete?: (() => void) | null\n): MonoTypeOperatorFunction<T> {\n  // We have to check to see not only if next is a function,\n  // but if error or complete were passed. This is because someone\n  // could technically call tap like `tap(null, fn)` or `tap(null, null, fn)`.\n  const tapObserver =\n    isFunction(observerOrNext) || error || complete\n      ? // tslint:disable-next-line: no-object-literal-type-assertion\n        ({ next: observerOrNext as Exclude<typeof observerOrNext, Partial<TapObserver<T>>>, error, complete } as Partial<TapObserver<T>>)\n      : observerOrNext;\n\n  return tapObserver\n    ? operate((source, subscriber) => {\n        tapObserver.subscribe?.();\n        let isUnsub = true;\n        source.subscribe(\n          createOperatorSubscriber(\n            subscriber,\n            (value) => {\n              tapObserver.next?.(value);\n              subscriber.next(value);\n            },\n            () => {\n              isUnsub = false;\n              tapObserver.complete?.();\n              subscriber.complete();\n            },\n            (err) => {\n              isUnsub = false;\n              tapObserver.error?.(err);\n              subscriber.error(err);\n            },\n            () => {\n              if (isUnsub) {\n                tapObserver.unsubscribe?.();\n              }\n              tapObserver.finalize?.();\n            }\n          )\n        );\n      })\n    : // Tap was called with no valid tap observer or handler\n      // (e.g. `tap(null, null, null)` or `tap(null)` or `tap()`)\n      // so we're going to just mirror the source.\n      identity;\n}\n", "import { Subscription } from '../Subscription';\n\nimport { MonoTypeOperatorFunction, ObservableInput } from '../types';\nimport { operate } from '../util/lift';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\nimport { innerFrom } from '../observable/innerFrom';\n\n/**\n * An object interface used by {@link throttle} or {@link throttleTime} that ensure\n * configuration options of these operators.\n *\n * @see {@link throttle}\n * @see {@link throttleTime}\n */\nexport interface ThrottleConfig {\n  /**\n   * If `true`, the resulting Observable will emit the first value from the source\n   * Observable at the **start** of the \"throttling\" process (when starting an\n   * internal timer that prevents other emissions from the source to pass through).\n   * If `false`, it will not emit the first value from the source Observable at the\n   * start of the \"throttling\" process.\n   *\n   * If not provided, defaults to: `true`.\n   */\n  leading?: boolean;\n  /**\n   * If `true`, the resulting Observable will emit the last value from the source\n   * Observable at the **end** of the \"throttling\" process (when ending an internal\n   * timer that prevents other emissions from the source to pass through).\n   * If `false`, it will not emit the last value from the source Observable at the\n   * end of the \"throttling\" process.\n   *\n   * If not provided, defaults to: `false`.\n   */\n  trailing?: boolean;\n}\n\n/**\n * Emits a value from the source Observable, then ignores subsequent source\n * values for a duration determined by another Observable, then repeats this\n * process.\n *\n * <span class=\"informal\">It's like {@link throttleTime}, but the silencing\n * duration is determined by a second Observable.</span>\n *\n * ![](throttle.svg)\n *\n * `throttle` emits the source Observable values on the output Observable\n * when its internal timer is disabled, and ignores source values when the timer\n * is enabled. Initially, the timer is disabled. As soon as the first source\n * value arrives, it is forwarded to the output Observable, and then the timer\n * is enabled by calling the `durationSelector` function with the source value,\n * which returns the \"duration\" Observable. When the duration Observable emits a\n * value, the timer is disabled, and this process repeats for the\n * next source value.\n *\n * ## Example\n *\n * Emit clicks at a rate of at most one click per second\n *\n * ```ts\n * import { fromEvent, throttle, interval } from 'rxjs';\n *\n * const clicks = fromEvent(document, 'click');\n * const result = clicks.pipe(throttle(() => interval(1000)));\n *\n * result.subscribe(x => console.log(x));\n * ```\n *\n * @see {@link audit}\n * @see {@link debounce}\n * @see {@link delayWhen}\n * @see {@link sample}\n * @see {@link throttleTime}\n *\n * @param durationSelector A function that receives a value from the source\n * Observable, for computing the silencing duration for each source value,\n * returned as an `ObservableInput`.\n * @param config A configuration object to define `leading` and `trailing`\n * behavior. Defaults to `{ leading: true, trailing: false }`.\n * @return A function that returns an Observable that performs the throttle\n * operation to limit the rate of emissions from the source.\n */\nexport function throttle<T>(durationSelector: (value: T) => ObservableInput<any>, config?: ThrottleConfig): MonoTypeOperatorFunction<T> {\n  return operate((source, subscriber) => {\n    const { leading = true, trailing = false } = config ?? {};\n    let hasValue = false;\n    let sendValue: T | null = null;\n    let throttled: Subscription | null = null;\n    let isComplete = false;\n\n    const endThrottling = () => {\n      throttled?.unsubscribe();\n      throttled = null;\n      if (trailing) {\n        send();\n        isComplete && subscriber.complete();\n      }\n    };\n\n    const cleanupThrottling = () => {\n      throttled = null;\n      isComplete && subscriber.complete();\n    };\n\n    const startThrottle = (value: T) =>\n      (throttled = innerFrom(durationSelector(value)).subscribe(createOperatorSubscriber(subscriber, endThrottling, cleanupThrottling)));\n\n    const send = () => {\n      if (hasValue) {\n        // Ensure we clear out our value and hasValue flag\n        // before we emit, otherwise reentrant code can cause\n        // issues here.\n        hasValue = false;\n        const value = sendValue!;\n        sendValue = null;\n        // Emit the value.\n        subscriber.next(value);\n        !isComplete && startThrottle(value);\n      }\n    };\n\n    source.subscribe(\n      createOperatorSubscriber(\n        subscriber,\n        // Regarding the presence of throttled.closed in the following\n        // conditions, if a synchronous duration selector is specified - weird,\n        // but legal - an already-closed subscription will be assigned to\n        // throttled, so the subscription's closed property needs to be checked,\n        // too.\n        (value) => {\n          hasValue = true;\n          sendValue = value;\n          !(throttled && !throttled.closed) && (leading ? send() : startThrottle(value));\n        },\n        () => {\n          isComplete = true;\n          !(trailing && hasValue && throttled && !throttled.closed) && subscriber.complete();\n        }\n      )\n    );\n  });\n}\n", "import { asyncScheduler } from '../scheduler/async';\nimport { throttle, ThrottleConfig } from './throttle';\nimport { MonoTypeOperatorFunction, SchedulerLike } from '../types';\nimport { timer } from '../observable/timer';\n\n/**\n * Emits a value from the source Observable, then ignores subsequent source\n * values for `duration` milliseconds, then repeats this process.\n *\n * <span class=\"informal\">Lets a value pass, then ignores source values for the\n * next `duration` milliseconds.</span>\n *\n * ![](throttleTime.png)\n *\n * `throttleTime` emits the source Observable values on the output Observable\n * when its internal timer is disabled, and ignores source values when the timer\n * is enabled. Initially, the timer is disabled. As soon as the first source\n * value arrives, it is forwarded to the output Observable, and then the timer\n * is enabled. After `duration` milliseconds (or the time unit determined\n * internally by the optional `scheduler`) has passed, the timer is disabled,\n * and this process repeats for the next source value. Optionally takes a\n * {@link SchedulerLike} for managing timers.\n *\n * ## Examples\n *\n * ### Limit click rate\n *\n * Emit clicks at a rate of at most one click per second\n *\n * ```ts\n * import { fromEvent, throttleTime } from 'rxjs';\n *\n * const clicks = fromEvent(document, 'click');\n * const result = clicks.pipe(throttleTime(1000));\n *\n * result.subscribe(x => console.log(x));\n * ```\n *\n * @see {@link auditTime}\n * @see {@link debounceTime}\n * @see {@link delay}\n * @see {@link sampleTime}\n * @see {@link throttle}\n *\n * @param duration Time to wait before emitting another value after\n * emitting the last value, measured in milliseconds or the time unit determined\n * internally by the optional `scheduler`.\n * @param scheduler The {@link SchedulerLike} to use for\n * managing the timers that handle the throttling. Defaults to {@link asyncScheduler}.\n * @param config A configuration object to define `leading` and\n * `trailing` behavior. Defaults to `{ leading: true, trailing: false }`.\n * @return A function that returns an Observable that performs the throttle\n * operation to limit the rate of emissions from the source.\n */\nexport function throttleTime<T>(\n  duration: number,\n  scheduler: SchedulerLike = asyncScheduler,\n  config?: ThrottleConfig\n): MonoTypeOperatorFunction<T> {\n  const duration$ = timer(duration, scheduler);\n  return throttle(() => duration$, config);\n}\n", "import { OperatorFunction, ObservableInputTuple } from '../types';\nimport { operate } from '../util/lift';\nimport { createOperatorSubscriber } from './OperatorSubscriber';\nimport { innerFrom } from '../observable/innerFrom';\nimport { identity } from '../util/identity';\nimport { noop } from '../util/noop';\nimport { popResultSelector } from '../util/args';\n\nexport function withLatestFrom<T, O extends unknown[]>(...inputs: [...ObservableInputTuple<O>]): OperatorFunction<T, [T, ...O]>;\n\nexport function withLatestFrom<T, O extends unknown[], R>(\n  ...inputs: [...ObservableInputTuple<O>, (...value: [T, ...O]) => R]\n): OperatorFunction<T, R>;\n\n/**\n * Combines the source Observable with other Observables to create an Observable\n * whose values are calculated from the latest values of each, only when the\n * source emits.\n *\n * <span class=\"informal\">Whenever the source Observable emits a value, it\n * computes a formula using that value plus the latest values from other input\n * Observables, then emits the output of that formula.</span>\n *\n * ![](withLatestFrom.png)\n *\n * `withLatestFrom` combines each value from the source Observable (the\n * instance) with the latest values from the other input Observables only when\n * the source emits a value, optionally using a `project` function to determine\n * the value to be emitted on the output Observable. All input Observables must\n * emit at least one value before the output Observable will emit a value.\n *\n * ## Example\n *\n * On every click event, emit an array with the latest timer event plus the click event\n *\n * ```ts\n * import { fromEvent, interval, withLatestFrom } from 'rxjs';\n *\n * const clicks = fromEvent(document, 'click');\n * const timer = interval(1000);\n * const result = clicks.pipe(withLatestFrom(timer));\n * result.subscribe(x => console.log(x));\n * ```\n *\n * @see {@link combineLatest}\n *\n * @param {ObservableInput} other An input Observable to combine with the source\n * Observable. More than one input Observables may be given as argument.\n * @param {Function} [project] Projection function for combining values\n * together. Receives all values in order of the Observables passed, where the\n * first parameter is a value from the source Observable. (e.g.\n * `a.pipe(withLatestFrom(b, c), map(([a1, b1, c1]) => a1 + b1 + c1))`). If this is not\n * passed, arrays will be emitted on the output Observable.\n * @return A function that returns an Observable of projected values from the\n * most recent values from each input Observable, or an array of the most\n * recent values from each input Observable.\n */\nexport function withLatestFrom<T, R>(...inputs: any[]): OperatorFunction<T, R | any[]> {\n  const project = popResultSelector(inputs) as ((...args: any[]) => R) | undefined;\n\n  return operate((source, subscriber) => {\n    const len = inputs.length;\n    const otherValues = new Array(len);\n    // An array of whether or not the other sources have emitted. Matched with them by index.\n    // TODO: At somepoint, we should investigate the performance implications here, and look\n    // into using a `Set()` and checking the `size` to see if we're ready.\n    let hasValue = inputs.map(() => false);\n    // Flipped true when we have at least one value from all other sources and\n    // we are ready to start emitting values.\n    let ready = false;\n\n    // Other sources. Note that here we are not checking `subscriber.closed`,\n    // this causes all inputs to be subscribed to, even if nothing can be emitted\n    // from them. This is an important distinction because subscription constitutes\n    // a side-effect.\n    for (let i = 0; i < len; i++) {\n      innerFrom(inputs[i]).subscribe(\n        createOperatorSubscriber(\n          subscriber,\n          (value) => {\n            otherValues[i] = value;\n            if (!ready && !hasValue[i]) {\n              // If we're not ready yet, flag to show this observable has emitted.\n              hasValue[i] = true;\n              // Intentionally terse code.\n              // If all of our other observables have emitted, set `ready` to `true`,\n              // so we know we can start emitting values, then clean up the `hasValue` array,\n              // because we don't need it anymore.\n              (ready = hasValue.every(identity)) && (hasValue = null!);\n            }\n          },\n          // Completing one of the other sources has\n          // no bearing on the completion of our result.\n          noop\n        )\n      );\n    }\n\n    // Source subscription\n    source.subscribe(\n      createOperatorSubscriber(subscriber, (value) => {\n        if (ready) {\n          // We have at least one value from the other sources. Go ahead and emit.\n          const values = [value, ...otherValues];\n          subscriber.next(project ? project(...values) : values);\n        }\n      })\n    );\n  });\n}\n", "import { zip as zipStatic } from '../observable/zip';\nimport { ObservableInput, ObservableInputTuple, OperatorFunction, Cons } from '../types';\nimport { operate } from '../util/lift';\n\n/** @deprecated Replaced with {@link zipWith}. Will be removed in v8. */\nexport function zip<T, A extends readonly unknown[]>(otherInputs: [...ObservableInputTuple<A>]): OperatorFunction<T, Cons<T, A>>;\n/** @deprecated Replaced with {@link zipWith}. Will be removed in v8. */\nexport function zip<T, A extends readonly unknown[], R>(\n  otherInputsAndProject: [...ObservableInputTuple<A>],\n  project: (...values: Cons<T, A>) => R\n): OperatorFunction<T, R>;\n/** @deprecated Replaced with {@link zipWith}. Will be removed in v8. */\nexport function zip<T, A extends readonly unknown[]>(...otherInputs: [...ObservableInputTuple<A>]): OperatorFunction<T, Cons<T, A>>;\n/** @deprecated Replaced with {@link zipWith}. Will be removed in v8. */\nexport function zip<T, A extends readonly unknown[], R>(\n  ...otherInputsAndProject: [...ObservableInputTuple<A>, (...values: Cons<T, A>) => R]\n): OperatorFunction<T, R>;\n\n/**\n * @deprecated Replaced with {@link zipWith}. Will be removed in v8.\n */\nexport function zip<T, R>(...sources: Array<ObservableInput<any> | ((...values: Array<any>) => R)>): OperatorFunction<T, any> {\n  return operate((source, subscriber) => {\n    zipStatic(source as ObservableInput<any>, ...(sources as Array<ObservableInput<any>>)).subscribe(subscriber);\n  });\n}\n", "import { ObservableInputTuple, OperatorFunction, Cons } from '../types';\nimport { zip } from './zip';\n\n/**\n * Subscribes to the source, and the observable inputs provided as arguments, and combines their values, by index, into arrays.\n *\n * What is meant by \"combine by index\": The first value from each will be made into a single array, then emitted,\n * then the second value from each will be combined into a single array and emitted, then the third value\n * from each will be combined into a single array and emitted, and so on.\n *\n * This will continue until it is no longer able to combine values of the same index into an array.\n *\n * After the last value from any one completed source is emitted in an array, the resulting observable will complete,\n * as there is no way to continue \"zipping\" values together by index.\n *\n * Use-cases for this operator are limited. There are memory concerns if one of the streams is emitting\n * values at a much faster rate than the others. Usage should likely be limited to streams that emit\n * at a similar pace, or finite streams of known length.\n *\n * In many cases, authors want `combineLatestWith` and not `zipWith`.\n *\n * @param otherInputs other observable inputs to collate values from.\n * @return A function that returns an Observable that emits items by index\n * combined from the source Observable and provided Observables, in form of an\n * array.\n */\nexport function zipWith<T, A extends readonly unknown[]>(...otherInputs: [...ObservableInputTuple<A>]): OperatorFunction<T, Cons<T, A>> {\n  return zip(...otherInputs);\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  ReplaySubject,\n  Subject,\n  fromEvent\n} from \"rxjs\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch document\n *\n * Documents are implemented as subjects, so all downstream observables are\n * automatically updated when a new document is emitted.\n *\n * @returns Document subject\n */\nexport function watchDocument(): Subject<Document> {\n  const document$ = new ReplaySubject<Document>(1)\n  fromEvent(document, \"DOMContentLoaded\", { once: true })\n    .subscribe(() => document$.next(document))\n\n  /* Return document */\n  return document$\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve all elements matching the query selector\n *\n * @template T - Element type\n *\n * @param selector - Query selector\n * @param node - Node of reference\n *\n * @returns Elements\n */\nexport function getElements<T extends keyof HTMLElementTagNameMap>(\n  selector: T, node?: ParentNode\n): HTMLElementTagNameMap[T][]\n\nexport function getElements<T extends HTMLElement>(\n  selector: string, node?: ParentNode\n): T[]\n\nexport function getElements<T extends HTMLElement>(\n  selector: string, node: ParentNode = document\n): T[] {\n  return Array.from(node.querySelectorAll<T>(selector))\n}\n\n/**\n * Retrieve an element matching a query selector or throw a reference error\n *\n * Note that this function assumes that the element is present. If unsure if an\n * element is existent, use the `getOptionalElement` function instead.\n *\n * @template T - Element type\n *\n * @param selector - Query selector\n * @param node - Node of reference\n *\n * @returns Element\n */\nexport function getElement<T extends keyof HTMLElementTagNameMap>(\n  selector: T, node?: ParentNode\n): HTMLElementTagNameMap[T]\n\nexport function getElement<T extends HTMLElement>(\n  selector: string, node?: ParentNode\n): T\n\nexport function getElement<T extends HTMLElement>(\n  selector: string, node: ParentNode = document\n): T {\n  const el = getOptionalElement<T>(selector, node)\n  if (typeof el === \"undefined\")\n    throw new ReferenceError(\n      `Missing element: expected \"${selector}\" to be present`\n    )\n\n  /* Return element */\n  return el\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Retrieve an optional element matching the query selector\n *\n * @template T - Element type\n *\n * @param selector - Query selector\n * @param node - Node of reference\n *\n * @returns Element or nothing\n */\nexport function getOptionalElement<T extends keyof HTMLElementTagNameMap>(\n  selector: T, node?: ParentNode\n): HTMLElementTagNameMap[T] | undefined\n\nexport function getOptionalElement<T extends HTMLElement>(\n  selector: string, node?: ParentNode\n): T | undefined\n\nexport function getOptionalElement<T extends HTMLElement>(\n  selector: string, node: ParentNode = document\n): T | undefined {\n  return node.querySelector<T>(selector) || undefined\n}\n\n/**\n * Retrieve the currently active element\n *\n * @returns Element or nothing\n */\nexport function getActiveElement(): HTMLElement | undefined {\n  return (\n    document.activeElement?.shadowRoot?.activeElement as HTMLElement ??\n    document.activeElement as HTMLElement ??\n    undefined\n  )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  debounceTime,\n  distinctUntilChanged,\n  fromEvent,\n  map,\n  merge,\n  shareReplay,\n  startWith\n} from \"rxjs\"\n\nimport { getActiveElement } from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Data\n * ------------------------------------------------------------------------- */\n\n/**\n * Focus observable\n *\n * Previously, this observer used `focus` and `blur` events to determine whether\n * an element is focused, but this doesn't work if there are focusable elements\n * within the elements itself. A better solutions are `focusin` and `focusout`\n * events, which bubble up the tree and allow for more fine-grained control.\n *\n * `debounceTime` is necessary, because when a focus change happens inside an\n * element, the observable would first emit `false` and then `true` again.\n */\nconst observer$ = merge(\n  fromEvent(document.body, \"focusin\"),\n  fromEvent(document.body, \"focusout\")\n)\n  .pipe(\n    debounceTime(1),\n    startWith(undefined),\n    map(() => getActiveElement() || document.body),\n    shareReplay(1)\n  )\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch element focus\n *\n * @param el - Element\n *\n * @returns Element focus observable\n */\nexport function watchElementFocus(\n  el: HTMLElement\n): Observable<boolean> {\n  return observer$\n    .pipe(\n      map(active => el.contains(active)),\n      distinctUntilChanged()\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  debounceTime,\n  fromEvent,\n  identity,\n  map,\n  merge,\n  startWith\n} from \"rxjs\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch element hover\n *\n * @param el - Element\n * @param duration - Debounce duration\n *\n * @returns Element hover observable\n */\nexport function watchElementHover(\n  el: HTMLElement, duration?: number\n): Observable<boolean> {\n  return merge(\n    fromEvent(el, \"mouseenter\").pipe(map(() => true)),\n    fromEvent(el, \"mouseleave\").pipe(map(() => false))\n  )\n    .pipe(\n      duration ? debounceTime(duration) : identity,\n      startWith(false)\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  animationFrameScheduler,\n  auditTime,\n  fromEvent,\n  map,\n  merge,\n  startWith\n} from \"rxjs\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Element offset\n */\nexport interface ElementOffset {\n  x: number                            /* Horizontal offset */\n  y: number                            /* Vertical offset */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve element offset\n *\n * @param el - Element\n *\n * @returns Element offset\n */\nexport function getElementOffset(\n  el: HTMLElement\n): ElementOffset {\n  return {\n    x: el.offsetLeft,\n    y: el.offsetTop\n  }\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Watch element offset\n *\n * @param el - Element\n *\n * @returns Element offset observable\n */\nexport function watchElementOffset(\n  el: HTMLElement\n): Observable<ElementOffset> {\n  return merge(\n    fromEvent(window, \"load\"),\n    fromEvent(window, \"resize\")\n  )\n    .pipe(\n      auditTime(0, animationFrameScheduler),\n      map(() => getElementOffset(el)),\n      startWith(getElementOffset(el))\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  animationFrameScheduler,\n  auditTime,\n  fromEvent,\n  map,\n  merge,\n  startWith\n} from \"rxjs\"\n\nimport { ElementOffset } from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve element content offset (= scroll offset)\n *\n * @param el - Element\n *\n * @returns Element content offset\n */\nexport function getElementContentOffset(\n  el: HTMLElement\n): ElementOffset {\n  return {\n    x: el.scrollLeft,\n    y: el.scrollTop\n  }\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Watch element content offset\n *\n * @param el - Element\n *\n * @returns Element content offset observable\n */\nexport function watchElementContentOffset(\n  el: HTMLElement\n): Observable<ElementOffset> {\n  return merge(\n    fromEvent(el, \"scroll\"),\n    fromEvent(window, \"resize\")\n  )\n    .pipe(\n      auditTime(0, animationFrameScheduler),\n      map(() => getElementContentOffset(el)),\n      startWith(getElementContentOffset(el))\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { JSX as JSXInternal } from \"preact\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * HTML attributes\n */\ntype Attributes =\n  & JSXInternal.HTMLAttributes\n  & JSXInternal.SVGAttributes\n  & Record<string, any>\n\n/**\n * Child element\n */\ntype Child =\n  | ChildNode\n  | HTMLElement\n  | Text\n  | string\n  | number\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Append a child node to an element\n *\n * @param el - Element\n * @param child - Child node(s)\n */\nfunction appendChild(el: HTMLElement, child: Child | Child[]): void {\n\n  /* Handle primitive types (including raw HTML) */\n  if (typeof child === \"string\" || typeof child === \"number\") {\n    el.innerHTML += child.toString()\n\n  /* Handle nodes */\n  } else if (child instanceof Node) {\n    el.appendChild(child)\n\n  /* Handle nested children */\n  } else if (Array.isArray(child)) {\n    for (const node of child)\n      appendChild(el, node)\n  }\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * JSX factory\n *\n * @template T - Element type\n *\n * @param tag - HTML tag\n * @param attributes - HTML attributes\n * @param children - Child elements\n *\n * @returns Element\n */\nexport function h<T extends keyof HTMLElementTagNameMap>(\n  tag: T, attributes?: Attributes | null, ...children: Child[]\n): HTMLElementTagNameMap[T]\n\nexport function h<T extends h.JSX.Element>(\n  tag: string, attributes?: Attributes | null, ...children: Child[]\n): T\n\nexport function h<T extends h.JSX.Element>(\n  tag: string, attributes?: Attributes | null, ...children: Child[]\n): T {\n  const el = document.createElement(tag)\n\n  /* Set attributes, if any */\n  if (attributes)\n    for (const attr of Object.keys(attributes)) {\n      if (typeof attributes[attr] === \"undefined\")\n        continue\n\n      /* Set default attribute or boolean */\n      if (typeof attributes[attr] !== \"boolean\")\n        el.setAttribute(attr, attributes[attr])\n      else\n        el.setAttribute(attr, \"\")\n    }\n\n  /* Append child nodes */\n  for (const child of children)\n    appendChild(el, child)\n\n  /* Return element */\n  return el as T\n}\n\n/* ----------------------------------------------------------------------------\n * Namespace\n * ------------------------------------------------------------------------- */\n\nexport declare namespace h {\n  namespace JSX {\n    type Element = HTMLElement\n    type IntrinsicElements = JSXInternal.IntrinsicElements\n  }\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Round a number for display with repository facts\n *\n * This is a reverse-engineered version of GitHub's weird rounding algorithm\n * for stars, forks and all other numbers. While all numbers below `1,000` are\n * returned as-is, bigger numbers are converted to fixed numbers:\n *\n * - `1,049` => `1k`\n * - `1,050` => `1.1k`\n * - `1,949` => `1.9k`\n * - `1,950` => `2k`\n *\n * @param value - Original value\n *\n * @returns Rounded value\n */\nexport function round(value: number): string {\n  if (value > 999) {\n    const digits = +((value - 950) % 1000 > 99)\n    return `${((value + 0.000001) / 1000).toFixed(digits)}k`\n  } else {\n    return value.toString()\n  }\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  defer,\n  finalize,\n  fromEvent,\n  map,\n  merge,\n  switchMap,\n  take,\n  throwError\n} from \"rxjs\"\n\nimport { h } from \"~/utilities\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Create and load a `script` element\n *\n * This function returns an observable that will emit when the script was\n * successfully loaded, or throw an error if it wasn't.\n *\n * @param src - Script URL\n *\n * @returns Script observable\n */\nexport function watchScript(src: string): Observable<void> {\n  const script = h(\"script\", { src })\n  return defer(() => {\n    document.head.appendChild(script)\n    return merge(\n      fromEvent(script, \"load\"),\n      fromEvent(script, \"error\")\n        .pipe(\n          switchMap(() => (\n            throwError(() => new ReferenceError(`Invalid script: ${src}`))\n          ))\n        )\n    )\n      .pipe(\n        map(() => undefined),\n        finalize(() => document.head.removeChild(script)),\n        take(1)\n      )\n  })\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  NEVER,\n  Observable,\n  Subject,\n  defer,\n  filter,\n  finalize,\n  map,\n  merge,\n  of,\n  shareReplay,\n  startWith,\n  switchMap,\n  tap\n} from \"rxjs\"\n\nimport { watchScript } from \"../../../script\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Element offset\n */\nexport interface ElementSize {\n  width: number                        /* Element width */\n  height: number                       /* Element height */\n}\n\n/* ----------------------------------------------------------------------------\n * Data\n * ------------------------------------------------------------------------- */\n\n/**\n * Resize observer entry subject\n */\nconst entry$ = new Subject<ResizeObserverEntry>()\n\n/**\n * Resize observer observable\n *\n * This observable will create a `ResizeObserver` on the first subscription\n * and will automatically terminate it when there are no more subscribers.\n * It's quite important to centralize observation in a single `ResizeObserver`,\n * as the performance difference can be quite dramatic, as the link shows.\n *\n * If the browser doesn't have a `ResizeObserver` implementation available, a\n * polyfill is automatically downloaded from unpkg.com. This is also compatible\n * with the built-in privacy plugin, which will download the polyfill and put\n * it alongside the built site for self-hosting.\n *\n * @see https://bit.ly/3iIYfEm - Google Groups on performance\n */\nconst observer$ = defer(() => (\n  typeof ResizeObserver === \"undefined\"\n    ? watchScript(\"https://unpkg.com/resize-observer-polyfill\")\n    : of(undefined)\n))\n  .pipe(\n    map(() => new ResizeObserver(entries => {\n      for (const entry of entries)\n        entry$.next(entry)\n    })),\n    switchMap(observer => merge(NEVER, of(observer))\n      .pipe(\n        finalize(() => observer.disconnect())\n      )\n    ),\n    shareReplay(1)\n  )\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve element size\n *\n * @param el - Element\n *\n * @returns Element size\n */\nexport function getElementSize(\n  el: HTMLElement\n): ElementSize {\n  return {\n    width:  el.offsetWidth,\n    height: el.offsetHeight\n  }\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Watch element size\n *\n * This function returns an observable that subscribes to a single internal\n * instance of `ResizeObserver` upon subscription, and emit resize events until\n * termination. Note that this function should not be called with the same\n * element twice, as the first unsubscription will terminate observation.\n *\n * Sadly, we can't use the `DOMRect` objects returned by the observer, because\n * we need the emitted values to be consistent with `getElementSize`, which will\n * return the used values (rounded) and not actual values (unrounded). Thus, we\n * use the `offset*` properties. See the linked GitHub issue.\n *\n * @see https://bit.ly/3m0k3he - GitHub issue\n *\n * @param el - Element\n *\n * @returns Element size observable\n */\nexport function watchElementSize(\n  el: HTMLElement\n): Observable<ElementSize> {\n  return observer$\n    .pipe(\n      tap(observer => observer.observe(el)),\n      switchMap(observer => entry$\n        .pipe(\n          filter(({ target }) => target === el),\n          finalize(() => observer.unobserve(el)),\n          map(() => getElementSize(el))\n        )\n      ),\n      startWith(getElementSize(el))\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { ElementSize } from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve element content size (= scroll width and height)\n *\n * @param el - Element\n *\n * @returns Element content size\n */\nexport function getElementContentSize(\n  el: HTMLElement\n): ElementSize {\n  return {\n    width:  el.scrollWidth,\n    height: el.scrollHeight\n  }\n}\n\n/**\n * Retrieve the overflowing container of an element, if any\n *\n * @param el - Element\n *\n * @returns Overflowing container or nothing\n */\nexport function getElementContainer(\n  el: HTMLElement\n): HTMLElement | undefined {\n  let parent = el.parentElement\n  while (parent)\n    if (\n      el.scrollWidth  <= parent.scrollWidth &&\n      el.scrollHeight <= parent.scrollHeight\n    )\n      parent = (el = parent).parentElement\n    else\n      break\n\n  /* Return overflowing container */\n  return parent ? el : undefined\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  NEVER,\n  Observable,\n  Subject,\n  defer,\n  distinctUntilChanged,\n  filter,\n  finalize,\n  map,\n  merge,\n  of,\n  shareReplay,\n  switchMap,\n  tap\n} from \"rxjs\"\n\nimport {\n  getElementContentSize,\n  getElementSize,\n  watchElementContentOffset\n} from \"~/browser\"\n\n/* ----------------------------------------------------------------------------\n * Data\n * ------------------------------------------------------------------------- */\n\n/**\n * Intersection observer entry subject\n */\nconst entry$ = new Subject<IntersectionObserverEntry>()\n\n/**\n * Intersection observer observable\n *\n * This observable will create an `IntersectionObserver` on first subscription\n * and will automatically terminate it when there are no more subscribers.\n *\n * @see https://bit.ly/3iIYfEm - Google Groups on performance\n */\nconst observer$ = defer(() => of(\n  new IntersectionObserver(entries => {\n    for (const entry of entries)\n      entry$.next(entry)\n  }, {\n    threshold: 0\n  })\n))\n  .pipe(\n    switchMap(observer => merge(NEVER, of(observer))\n      .pipe(\n        finalize(() => observer.disconnect())\n      )\n    ),\n    shareReplay(1)\n  )\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch element visibility\n *\n * @param el - Element\n *\n * @returns Element visibility observable\n */\nexport function watchElementVisibility(\n  el: HTMLElement\n): Observable<boolean> {\n  return observer$\n    .pipe(\n      tap(observer => observer.observe(el)),\n      switchMap(observer => entry$\n        .pipe(\n          filter(({ target }) => target === el),\n          finalize(() => observer.unobserve(el)),\n          map(({ isIntersecting }) => isIntersecting)\n        )\n      )\n    )\n}\n\n/**\n * Watch element boundary\n *\n * This function returns an observable which emits whether the bottom content\n * boundary (= scroll offset) of an element is within a certain threshold.\n *\n * @param el - Element\n * @param threshold - Threshold\n *\n * @returns Element boundary observable\n */\nexport function watchElementBoundary(\n  el: HTMLElement, threshold = 16\n): Observable<boolean> {\n  return watchElementContentOffset(el)\n    .pipe(\n      map(({ y }) => {\n        const visible = getElementSize(el)\n        const content = getElementContentSize(el)\n        return y >= (\n          content.height - visible.height - threshold\n        )\n      }),\n      distinctUntilChanged()\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  fromEvent,\n  map,\n  startWith\n} from \"rxjs\"\n\nimport { getElement } from \"../element\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Toggle\n */\nexport type Toggle =\n  | \"drawer\"                           /* Toggle for drawer */\n  | \"search\"                           /* Toggle for search */\n\n/* ----------------------------------------------------------------------------\n * Data\n * ------------------------------------------------------------------------- */\n\n/**\n * Toggle map\n */\nconst toggles: Record<Toggle, HTMLInputElement> = {\n  drawer: getElement(\"[data-md-toggle=drawer]\"),\n  search: getElement(\"[data-md-toggle=search]\")\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve the value of a toggle\n *\n * @param name - Toggle\n *\n * @returns Toggle value\n */\nexport function getToggle(name: Toggle): boolean {\n  return toggles[name].checked\n}\n\n/**\n * Set toggle\n *\n * Simulating a click event seems to be the most cross-browser compatible way\n * of changing the value while also emitting a `change` event. Before, Material\n * used `CustomEvent` to programmatically change the value of a toggle, but this\n * is a much simpler and cleaner solution which doesn't require a polyfill.\n *\n * @param name - Toggle\n * @param value - Toggle value\n */\nexport function setToggle(name: Toggle, value: boolean): void {\n  if (toggles[name].checked !== value)\n    toggles[name].click()\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Watch toggle\n *\n * @param name - Toggle\n *\n * @returns Toggle value observable\n */\nexport function watchToggle(name: Toggle): Observable<boolean> {\n  const el = toggles[name]\n  return fromEvent(el, \"change\")\n    .pipe(\n      map(() => el.checked),\n      startWith(el.checked)\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  EMPTY,\n  Observable,\n  filter,\n  fromEvent,\n  map,\n  merge,\n  share,\n  startWith,\n  switchMap\n} from \"rxjs\"\n\nimport { getActiveElement } from \"../element\"\nimport { getToggle } from \"../toggle\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Keyboard mode\n */\nexport type KeyboardMode =\n  | \"global\"                           /* Global */\n  | \"search\"                           /* Search is open */\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Keyboard\n */\nexport interface Keyboard {\n  mode: KeyboardMode                   /* Keyboard mode */\n  type: string                         /* Key type */\n  claim(): void                        /* Key claim */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Check whether an element may receive keyboard input\n *\n * @param el - Element\n * @param type - Key type\n *\n * @returns Test result\n */\nfunction isSusceptibleToKeyboard(\n  el: HTMLElement, type: string\n): boolean {\n  switch (el.constructor) {\n\n    /* Input elements */\n    case HTMLInputElement:\n      /* @ts-expect-error - omit unnecessary type cast */\n      if (el.type === \"radio\")\n        return /^Arrow/.test(type)\n      else\n        return true\n\n    /* Select element and textarea */\n    case HTMLSelectElement:\n    case HTMLTextAreaElement:\n      return true\n\n    /* Everything else */\n    default:\n      return el.isContentEditable\n  }\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch composition events\n *\n * @returns Composition observable\n */\nexport function watchComposition(): Observable<boolean> {\n  return merge(\n    fromEvent(window, \"compositionstart\").pipe(map(() => true)),\n    fromEvent(window, \"compositionend\").pipe(map(() => false))\n  )\n    .pipe(\n      startWith(false)\n    )\n}\n\n/**\n * Watch keyboard\n *\n * @returns Keyboard observable\n */\nexport function watchKeyboard(): Observable<Keyboard> {\n  const keyboard$ = fromEvent<KeyboardEvent>(window, \"keydown\")\n    .pipe(\n      filter(ev => !(ev.metaKey || ev.ctrlKey)),\n      map(ev => ({\n        mode: getToggle(\"search\") ? \"search\" : \"global\",\n        type: ev.key,\n        claim() {\n          ev.preventDefault()\n          ev.stopPropagation()\n        }\n      } as Keyboard)),\n      filter(({ mode, type }) => {\n        if (mode === \"global\") {\n          const active = getActiveElement()\n          if (typeof active !== \"undefined\")\n            return !isSusceptibleToKeyboard(active, type)\n        }\n        return true\n      }),\n      share()\n    )\n\n  /* Don't emit during composition events - see https://bit.ly/3te3Wl8 */\n  return watchComposition()\n    .pipe(\n      switchMap(active => !active ? keyboard$ : EMPTY)\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Subject } from \"rxjs\"\n\nimport { feature } from \"~/_\"\nimport { h } from \"~/utilities\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve location\n *\n * This function returns a `URL` object (and not `Location`) to normalize the\n * typings across the application. Furthermore, locations need to be tracked\n * without setting them and `Location` is a singleton which represents the\n * current location.\n *\n * @returns URL\n */\nexport function getLocation(): URL {\n  return new URL(location.href)\n}\n\n/**\n * Set location\n *\n * If instant navigation is enabled, this function creates a temporary anchor\n * element, sets the `href` attribute, appends it to the body, clicks it, and\n * then removes it again. The event will bubble up the DOM and trigger be\n * intercepted by the instant loading business logic.\n *\n * Note that we must append and remove the anchor element, or the event will\n * not bubble up the DOM, making it impossible to intercept it.\n *\n * @param url - URL to navigate to\n * @param navigate - Force navigation\n */\nexport function setLocation(\n  url: URL | HTMLLinkElement, navigate = false\n): void {\n  if (feature(\"navigation.instant\") && !navigate) {\n    const el = h(\"a\", { href: url.href })\n    document.body.appendChild(el)\n    el.click()\n    el.remove()\n\n  // If we're not using instant navigation, and the page should not be reloaded\n  // just instruct the browser to navigate to the given URL\n  } else {\n    location.href = url.href\n  }\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Watch location\n *\n * @returns Location subject\n */\nexport function watchLocation(): Subject<URL> {\n  return new Subject<URL>()\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  filter,\n  fromEvent,\n  map,\n  merge,\n  shareReplay,\n  startWith\n} from \"rxjs\"\n\nimport { getOptionalElement } from \"~/browser\"\nimport { h } from \"~/utilities\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve location hash\n *\n * @returns Location hash\n */\nexport function getLocationHash(): string {\n  return location.hash.slice(1)\n}\n\n/**\n * Set location hash\n *\n * Setting a new fragment identifier via `location.hash` will have no effect\n * if the value doesn't change. When a new fragment identifier is set, we want\n * the browser to target the respective element at all times, which is why we\n * use this dirty little trick.\n *\n * @param hash - Location hash\n */\nexport function setLocationHash(hash: string): void {\n  const el = h(\"a\", { href: hash })\n  el.addEventListener(\"click\", ev => ev.stopPropagation())\n  el.click()\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Watch location hash\n *\n * @param location$ - Location observable\n *\n * @returns Location hash observable\n */\nexport function watchLocationHash(\n  location$: Observable<URL>\n): Observable<string> {\n  return merge(\n    fromEvent<HashChangeEvent>(window, \"hashchange\"),\n    location$\n  )\n    .pipe(\n      map(getLocationHash),\n      startWith(getLocationHash()),\n      filter(hash => hash.length > 0),\n      shareReplay(1)\n    )\n}\n\n/**\n * Watch location target\n *\n * @param location$ - Location observable\n *\n * @returns Location target observable\n */\nexport function watchLocationTarget(\n  location$: Observable<URL>\n): Observable<HTMLElement> {\n  return watchLocationHash(location$)\n    .pipe(\n      map(id => getOptionalElement(`[id=\"${id}\"]`)!),\n      filter(el => typeof el !== \"undefined\")\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  EMPTY,\n  Observable,\n  fromEvent,\n  fromEventPattern,\n  map,\n  merge,\n  startWith,\n  switchMap\n} from \"rxjs\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch media query\n *\n * Note that although `MediaQueryList.addListener` is deprecated we have to\n * use it, because it's the only way to ensure proper downward compatibility.\n *\n * @see https://bit.ly/3dUBH2m - GitHub issue\n *\n * @param query - Media query\n *\n * @returns Media observable\n */\nexport function watchMedia(query: string): Observable<boolean> {\n  const media = matchMedia(query)\n  return fromEventPattern<boolean>(next => (\n    media.addListener(() => next(media.matches))\n  ))\n    .pipe(\n      startWith(media.matches)\n    )\n}\n\n/**\n * Watch print mode\n *\n * @returns Print observable\n */\nexport function watchPrint(): Observable<boolean> {\n  const media = matchMedia(\"print\")\n  return merge(\n    fromEvent(window, \"beforeprint\").pipe(map(() => true)),\n    fromEvent(window, \"afterprint\").pipe(map(() => false))\n  )\n    .pipe(\n      startWith(media.matches)\n    )\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Toggle an observable with a media observable\n *\n * @template T - Data type\n *\n * @param query$ - Media observable\n * @param factory - Observable factory\n *\n * @returns Toggled observable\n */\nexport function at<T>(\n  query$: Observable<boolean>, factory: () => Observable<T>\n): Observable<T> {\n  return query$\n    .pipe(\n      switchMap(active => active ? factory() : EMPTY)\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  map,\n  shareReplay,\n  switchMap\n} from \"rxjs\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Options\n */\ninterface Options {\n  progress$?: Subject<number>          // Progress subject\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Fetch the given URL\n *\n * This function returns an observable that emits the response as a blob and\n * completes, or emits an error if the request failed. The caller can cancel\n * the request by unsubscribing at any time, which will automatically abort\n * the inflight request and complete the observable.\n *\n * Note that we use `XMLHTTPRequest` not because we're nostalgic, but because\n * it's the only way to get progress events for downloads and also allow for\n * cancellation of requests, as the official Fetch API does not support this\n * yet, even though we're already in 2024.\n *\n * @param url - Request URL\n * @param options - Options\n *\n * @returns Data observable\n */\nexport function request(\n  url: URL | string, options?: Options\n): Observable<Blob> {\n  return new Observable<Blob>(observer => {\n    const req = new XMLHttpRequest()\n    req.open(\"GET\", `${url}`)\n    req.responseType = \"blob\"\n\n    // Handle response\n    req.addEventListener(\"load\", () => {\n      if (req.status >= 200 && req.status < 300) {\n        observer.next(req.response)\n        observer.complete()\n\n      // Every response that is not in the 2xx range is considered an error\n      } else {\n        observer.error(new Error(req.statusText))\n      }\n    })\n\n    // Handle network errors\n    req.addEventListener(\"error\", () => {\n      observer.error(new Error(\"Network error\"))\n    })\n\n    // Handle aborted requests\n    req.addEventListener(\"abort\", () => {\n      observer.complete()\n    })\n\n    // Handle download progress\n    if (typeof options?.progress$ !== \"undefined\") {\n      req.addEventListener(\"progress\", event => {\n        if (event.lengthComputable) {\n          options.progress$!.next((event.loaded / event.total) * 100)\n\n        // Hack: Chromium doesn't report the total number of bytes if content\n        // is compressed, so we need this fallback - see https://t.ly/ZXofI\n        } else {\n          const length = req.getResponseHeader(\"Content-Length\") ?? 0\n          options.progress$!.next((event.loaded / +length) * 100)\n        }\n      })\n\n      // Immediately set progress to 5% to indicate that we're loading\n      options.progress$.next(5)\n    }\n\n    // Send request and automatically abort request upon unsubscription\n    req.send()\n    return () => req.abort()\n  })\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Fetch JSON from the given URL\n *\n * @template T - Data type\n *\n * @param url - Request URL\n * @param options - Options\n *\n * @returns Data observable\n */\nexport function requestJSON<T>(\n  url: URL | string, options?: Options\n): Observable<T> {\n  return request(url, options)\n    .pipe(\n      switchMap(res => res.text()),\n      map(body => JSON.parse(body) as T),\n      shareReplay(1)\n    )\n}\n\n/**\n * Fetch HTML from the given URL\n *\n * @param url - Request URL\n * @param options - Options\n *\n * @returns Data observable\n */\nexport function requestHTML(\n  url: URL | string, options?: Options\n): Observable<Document> {\n  const dom = new DOMParser()\n  return request(url, options)\n    .pipe(\n      switchMap(res => res.text()),\n      map(res => dom.parseFromString(res, \"text/html\")),\n      shareReplay(1)\n    )\n}\n\n/**\n * Fetch XML from the given URL\n *\n * @param url - Request URL\n * @param options - Options\n *\n * @returns Data observable\n */\nexport function requestXML(\n  url: URL | string, options?: Options\n): Observable<Document> {\n  const dom = new DOMParser()\n  return request(url, options)\n    .pipe(\n      switchMap(res => res.text()),\n      map(res => dom.parseFromString(res, \"text/xml\")),\n      shareReplay(1)\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  fromEvent,\n  map,\n  merge,\n  startWith\n} from \"rxjs\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Viewport offset\n */\nexport interface ViewportOffset {\n  x: number                            /* Horizontal offset */\n  y: number                            /* Vertical offset */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve viewport offset\n *\n * On iOS Safari, viewport offset can be negative due to overflow scrolling.\n * As this may induce strange behaviors downstream, we'll just limit it to 0.\n *\n * @returns Viewport offset\n */\nexport function getViewportOffset(): ViewportOffset {\n  return {\n    x: Math.max(0, scrollX),\n    y: Math.max(0, scrollY)\n  }\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Watch viewport offset\n *\n * @returns Viewport offset observable\n */\nexport function watchViewportOffset(): Observable<ViewportOffset> {\n  return merge(\n    fromEvent(window, \"scroll\", { passive: true }),\n    fromEvent(window, \"resize\", { passive: true })\n  )\n    .pipe(\n      map(getViewportOffset),\n      startWith(getViewportOffset())\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  fromEvent,\n  map,\n  startWith\n} from \"rxjs\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Viewport size\n */\nexport interface ViewportSize {\n  width: number                        /* Viewport width */\n  height: number                       /* Viewport height */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve viewport size\n *\n * @returns Viewport size\n */\nexport function getViewportSize(): ViewportSize {\n  return {\n    width:  innerWidth,\n    height: innerHeight\n  }\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Watch viewport size\n *\n * @returns Viewport size observable\n */\nexport function watchViewportSize(): Observable<ViewportSize> {\n  return fromEvent(window, \"resize\", { passive: true })\n    .pipe(\n      map(getViewportSize),\n      startWith(getViewportSize())\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  combineLatest,\n  map,\n  shareReplay\n} from \"rxjs\"\n\nimport {\n  ViewportOffset,\n  watchViewportOffset\n} from \"../offset\"\nimport {\n  ViewportSize,\n  watchViewportSize\n} from \"../size\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Viewport\n */\nexport interface Viewport {\n  offset: ViewportOffset               /* Viewport offset */\n  size: ViewportSize                   /* Viewport size */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch viewport\n *\n * @returns Viewport observable\n */\nexport function watchViewport(): Observable<Viewport> {\n  return combineLatest([\n    watchViewportOffset(),\n    watchViewportSize()\n  ])\n    .pipe(\n      map(([offset, size]) => ({ offset, size })),\n      shareReplay(1)\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  combineLatest,\n  distinctUntilKeyChanged,\n  map\n} from \"rxjs\"\n\nimport { Header } from \"~/components\"\n\nimport { getElementOffset } from \"../../element\"\nimport { Viewport } from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch viewport relative to element\n *\n * @param el - Element\n * @param options - Options\n *\n * @returns Viewport observable\n */\nexport function watchViewportAt(\n  el: HTMLElement, { viewport$, header$ }: WatchOptions\n): Observable<Viewport> {\n  const size$ = viewport$\n    .pipe(\n      distinctUntilKeyChanged(\"size\")\n    )\n\n  /* Compute element offset */\n  const offset$ = combineLatest([size$, header$])\n    .pipe(\n      map(() => getElementOffset(el))\n    )\n\n  /* Compute relative viewport, return hot observable */\n  return combineLatest([header$, viewport$, offset$])\n    .pipe(\n      map(([{ height }, { offset, size }, { x, y }]) => ({\n        offset: {\n          x: offset.x - x,\n          y: offset.y - y + height\n        },\n        size\n      }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  endWith,\n  fromEvent,\n  ignoreElements,\n  mergeWith,\n  share,\n  takeUntil\n} from \"rxjs\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Worker message\n */\nexport interface WorkerMessage {\n  type: unknown                        /* Message type */\n  data?: unknown                       /* Message data */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Create an observable for receiving from a web worker\n *\n * @template T - Data type\n *\n * @param worker - Web worker\n *\n * @returns Message observable\n */\nfunction recv<T>(worker: Worker): Observable<T> {\n  return fromEvent<MessageEvent<T>, T>(worker, \"message\", ev => ev.data)\n}\n\n/**\n * Create a subject for sending to a web worker\n *\n * @template T - Data type\n *\n * @param worker - Web worker\n *\n * @returns Message subject\n */\nfunction send<T>(worker: Worker): Subject<T> {\n  const send$ = new Subject<T>()\n  send$.subscribe(data => worker.postMessage(data))\n\n  /* Return message subject */\n  return send$\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Create a bidirectional communication channel to a web worker\n *\n * @template T - Data type\n *\n * @param url - Worker URL\n * @param worker - Worker\n *\n * @returns Worker subject\n */\nexport function watchWorker<T extends WorkerMessage>(\n  url: string, worker = new Worker(url)\n): Subject<T> {\n  const recv$ = recv<T>(worker)\n  const send$ = send<T>(worker)\n\n  /* Create worker subject and forward messages */\n  const worker$ = new Subject<T>()\n  worker$.subscribe(send$)\n\n  /* Return worker subject */\n  const done$ = send$.pipe(ignoreElements(), endWith(true))\n  return worker$\n    .pipe(\n      ignoreElements(),\n      mergeWith(recv$.pipe(takeUntil(done$))),\n      share()\n    ) as Subject<T>\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { getElement, getLocation } from \"~/browser\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Feature flag\n */\nexport type Flag =\n  | \"announce.dismiss\"                 /* Dismissable announcement bar */\n  | \"content.code.annotate\"            /* Code annotations */\n  | \"content.code.copy\"                /* Code copy button */\n  | \"content.lazy\"                     /* Lazy content elements */\n  | \"content.tabs.link\"                /* Link content tabs */\n  | \"content.tooltips\"                 /* Tooltips */\n  | \"header.autohide\"                  /* Hide header */\n  | \"navigation.expand\"                /* Automatic expansion */\n  | \"navigation.indexes\"               /* Section pages */\n  | \"navigation.instant\"               /* Instant navigation */\n  | \"navigation.instant.progress\"      /* Instant navigation progress */\n  | \"navigation.sections\"              /* Section navigation */\n  | \"navigation.tabs\"                  /* Tabs navigation */\n  | \"navigation.tabs.sticky\"           /* Tabs navigation (sticky) */\n  | \"navigation.top\"                   /* Back-to-top button */\n  | \"navigation.tracking\"              /* Anchor tracking */\n  | \"search.highlight\"                 /* Search highlighting */\n  | \"search.share\"                     /* Search sharing */\n  | \"search.suggest\"                   /* Search suggestions */\n  | \"toc.follow\"                       /* Following table of contents */\n  | \"toc.integrate\"                    /* Integrated table of contents */\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Translation\n */\nexport type Translation =\n  | \"clipboard.copy\"                   /* Copy to clipboard */\n  | \"clipboard.copied\"                 /* Copied to clipboard */\n  | \"search.result.placeholder\"        /* Type to start searching */\n  | \"search.result.none\"               /* No matching documents */\n  | \"search.result.one\"                /* 1 matching document */\n  | \"search.result.other\"              /* # matching documents */\n  | \"search.result.more.one\"           /* 1 more on this page */\n  | \"search.result.more.other\"         /* # more on this page */\n  | \"search.result.term.missing\"       /* Missing */\n  | \"select.version\"                   /* Version selector */\n\n/**\n * Translations\n */\nexport type Translations =\n  Record<Translation, string>\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Versioning\n */\nexport interface Versioning {\n  provider: \"mike\"                     /* Version provider */\n  default?: string | string[]          /* Default version */\n}\n\n/**\n * Configuration\n */\nexport interface Config {\n  base: string                         /* Base URL */\n  features: Flag[]                     /* Feature flags */\n  translations: Translations           /* Translations */\n  search: string                       /* Search worker URL */\n  tags?: Record<string, string>        /* Tags mapping */\n  version?: Versioning                 /* Versioning */\n}\n\n/* ----------------------------------------------------------------------------\n * Data\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve global configuration and make base URL absolute\n */\nconst script = getElement(\"#__config\")\nconst config: Config = JSON.parse(script.textContent!)\nconfig.base = `${new URL(config.base, getLocation())}`\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve global configuration\n *\n * @returns Global configuration\n */\nexport function configuration(): Config {\n  return config\n}\n\n/**\n * Check whether a feature flag is enabled\n *\n * @param flag - Feature flag\n *\n * @returns Test result\n */\nexport function feature(flag: Flag): boolean {\n  return config.features.includes(flag)\n}\n\n/**\n * Retrieve the translation for the given key\n *\n * @param key - Key to be translated\n * @param value - Positional value, if any\n *\n * @returns Translation\n */\nexport function translation(\n  key: Translation, value?: string | number\n): string {\n  return typeof value !== \"undefined\"\n    ? config.translations[key].replace(\"#\", value.toString())\n    : config.translations[key]\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { getElement, getElements } from \"~/browser\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Component type\n */\nexport type ComponentType =\n  | \"announce\"                         /* Announcement bar */\n  | \"container\"                        /* Container */\n  | \"consent\"                          /* Consent */\n  | \"content\"                          /* Content */\n  | \"dialog\"                           /* Dialog */\n  | \"header\"                           /* Header */\n  | \"header-title\"                     /* Header title */\n  | \"header-topic\"                     /* Header topic */\n  | \"main\"                             /* Main area */\n  | \"outdated\"                         /* Version warning */\n  | \"palette\"                          /* Color palette */\n  | \"progress\"                         /* Progress indicator */\n  | \"search\"                           /* Search */\n  | \"search-query\"                     /* Search input */\n  | \"search-result\"                    /* Search results */\n  | \"search-share\"                     /* Search sharing */\n  | \"search-suggest\"                   /* Search suggestions */\n  | \"sidebar\"                          /* Sidebar */\n  | \"skip\"                             /* Skip link */\n  | \"source\"                           /* Repository information */\n  | \"tabs\"                             /* Navigation tabs */\n  | \"toc\"                              /* Table of contents */\n  | \"top\"                              /* Back-to-top button */\n\n/**\n * Component\n *\n * @template T - Component type\n * @template U - Reference type\n */\nexport type Component<\n  T extends {} = {},\n  U extends HTMLElement = HTMLElement\n> =\n  T & {\n    ref: U                             /* Component reference */\n  }\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Component type map\n */\ninterface ComponentTypeMap {\n  \"announce\": HTMLElement              /* Announcement bar */\n  \"container\": HTMLElement             /* Container */\n  \"consent\": HTMLElement               /* Consent */\n  \"content\": HTMLElement               /* Content */\n  \"dialog\": HTMLElement                /* Dialog */\n  \"header\": HTMLElement                /* Header */\n  \"header-title\": HTMLElement          /* Header title */\n  \"header-topic\": HTMLElement          /* Header topic */\n  \"main\": HTMLElement                  /* Main area */\n  \"outdated\": HTMLElement              /* Version warning */\n  \"palette\": HTMLElement               /* Color palette */\n  \"progress\": HTMLElement              /* Progress indicator */\n  \"search\": HTMLElement                /* Search */\n  \"search-query\": HTMLInputElement     /* Search input */\n  \"search-result\": HTMLElement         /* Search results */\n  \"search-share\": HTMLAnchorElement    /* Search sharing */\n  \"search-suggest\": HTMLElement        /* Search suggestions */\n  \"sidebar\": HTMLElement               /* Sidebar */\n  \"skip\": HTMLAnchorElement            /* Skip link */\n  \"source\": HTMLAnchorElement          /* Repository information */\n  \"tabs\": HTMLElement                  /* Navigation tabs */\n  \"toc\": HTMLElement                   /* Table of contents */\n  \"top\": HTMLAnchorElement             /* Back-to-top button */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve the element for a given component or throw a reference error\n *\n * @template T - Component type\n *\n * @param type - Component type\n * @param node - Node of reference\n *\n * @returns Element\n */\nexport function getComponentElement<T extends ComponentType>(\n  type: T, node: ParentNode = document\n): ComponentTypeMap[T] {\n  return getElement(`[data-md-component=${type}]`, node)\n}\n\n/**\n * Retrieve all elements for a given component\n *\n * @template T - Component type\n *\n * @param type - Component type\n * @param node - Node of reference\n *\n * @returns Elements\n */\nexport function getComponentElements<T extends ComponentType>(\n  type: T, node: ParentNode = document\n): ComponentTypeMap[T][] {\n  return getElements(`[data-md-component=${type}]`, node)\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  EMPTY,\n  Observable,\n  Subject,\n  defer,\n  finalize,\n  fromEvent,\n  map,\n  tap\n} from \"rxjs\"\n\nimport { feature } from \"~/_\"\nimport { getElement } from \"~/browser\"\n\nimport { Component } from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Announcement bar\n */\nexport interface Announce {\n  hash: number                        /* Content hash */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch announcement bar\n *\n * @param el - Announcement bar element\n *\n * @returns Announcement bar observable\n */\nexport function watchAnnounce(\n  el: HTMLElement\n): Observable<Announce> {\n  const button = getElement(\".md-typeset > :first-child\", el)\n  return fromEvent(button, \"click\", { once: true })\n    .pipe(\n      map(() => getElement(\".md-typeset\", el)),\n      map(content => ({ hash: __md_hash(content.innerHTML) }))\n    )\n}\n\n/**\n * Mount announcement bar\n *\n * @param el - Announcement bar element\n *\n * @returns Announcement bar component observable\n */\nexport function mountAnnounce(\n  el: HTMLElement\n): Observable<Component<Announce>> {\n  if (!feature(\"announce.dismiss\") || !el.childElementCount)\n    return EMPTY\n\n  /* Support instant navigation - see https://t.ly/3FTme */\n  if (!el.hidden) {\n    const content = getElement(\".md-typeset\", el)\n    if (__md_hash(content.innerHTML) === __md_get(\"__announce\"))\n      el.hidden = true\n  }\n\n  /* Mount component on subscription */\n  return defer(() => {\n    const push$ = new Subject<Announce>()\n    push$.subscribe(({ hash }) => {\n      el.hidden = true\n\n      /* Persist preference in local storage */\n      __md_set<number>(\"__announce\", hash)\n    })\n\n    /* Create and return component */\n    return watchAnnounce(el)\n      .pipe(\n        tap(state => push$.next(state)),\n        finalize(() => push$.complete()),\n        map(state => ({ ref: el, ...state }))\n      )\n  })\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  finalize,\n  map,\n  tap\n} from \"rxjs\"\n\nimport { Component } from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Consent\n */\nexport interface Consent {\n  hidden: boolean                      /* Consent is hidden */\n}\n\n/**\n * Consent defaults\n */\nexport interface ConsentDefaults {\n  analytics?: boolean                  /* Consent for Analytics */\n  github?: boolean                     /* Consent for GitHub */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  target$: Observable<HTMLElement>     /* Target observable */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  target$: Observable<HTMLElement>     /* Target observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch consent\n *\n * @param el - Consent element\n * @param options - Options\n *\n * @returns Consent observable\n */\nexport function watchConsent(\n  el: HTMLElement, { target$ }: WatchOptions\n): Observable<Consent> {\n  return target$\n    .pipe(\n      map(target => ({ hidden: target !== el }))\n    )\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Mount consent\n *\n * @param el - Consent element\n * @param options - Options\n *\n * @returns Consent component observable\n */\nexport function mountConsent(\n  el: HTMLElement, options: MountOptions\n): Observable<Component<Consent>> {\n  const internal$ = new Subject<Consent>()\n  internal$.subscribe(({ hidden }) => {\n    el.hidden = hidden\n  })\n\n  /* Create and return component */\n  return watchConsent(el, options)\n    .pipe(\n      tap(state => internal$.next(state)),\n      finalize(() => internal$.complete()),\n      map(state => ({ ref: el, ...state }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { h } from \"~/utilities\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Tooltip style\n */\nexport type TooltipStyle =\n  | \"inline\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Render a tooltip\n *\n * @param id - Tooltip identifier\n * @param style - Tooltip style\n *\n * @returns Element\n */\nexport function renderTooltip(\n  id?: string, style?: TooltipStyle\n): HTMLElement {\n  if (style === \"inline\") { // @todo refactor control flow\n    return (\n      <div class=\"md-tooltip md-tooltip--inline\" id={id} role=\"tooltip\">\n        <div class=\"md-tooltip__inner md-typeset\"></div>\n      </div>\n    )\n  } else {\n    return (\n      <div class=\"md-tooltip\" id={id} role=\"tooltip\">\n        <div class=\"md-tooltip__inner md-typeset\"></div>\n      </div>\n    )\n  }\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { h } from \"~/utilities\"\n\nimport { renderTooltip } from \"../tooltip\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Render an annotation\n *\n * @param id - Annotation identifier\n * @param prefix - Tooltip identifier prefix\n *\n * @returns Element\n */\nexport function renderAnnotation(\n  id: string | number, prefix?: string\n): HTMLElement {\n  prefix = prefix ? `${prefix}_annotation_${id}` : undefined\n\n  /* Render tooltip with anchor, if given */\n  if (prefix) {\n    const anchor = prefix ? `#${prefix}` : undefined\n    return (\n      <aside class=\"md-annotation\" tabIndex={0}>\n        {renderTooltip(prefix)}\n        <a href={anchor} class=\"md-annotation__index\" tabIndex={-1}>\n          <span data-md-annotation-id={id}></span>\n        </a>\n      </aside>\n    )\n  } else {\n    return (\n      <aside class=\"md-annotation\" tabIndex={0}>\n        {renderTooltip(prefix)}\n        <span class=\"md-annotation__index\" tabIndex={-1}>\n          <span data-md-annotation-id={id}></span>\n        </span>\n      </aside>\n    )\n  }\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { translation } from \"~/_\"\nimport { h } from \"~/utilities\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Render a 'copy-to-clipboard' button\n *\n * @param id - Unique identifier\n *\n * @returns Element\n */\nexport function renderClipboardButton(id: string): HTMLElement {\n  return (\n    <button\n      class=\"md-clipboard md-icon\"\n      title={translation(\"clipboard.copy\")}\n      data-clipboard-target={`#${id} > code`}\n    ></button>\n  )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { ComponentChild } from \"preact\"\n\nimport { configuration, feature, translation } from \"~/_\"\nimport { SearchItem } from \"~/integrations/search\"\nimport { h } from \"~/utilities\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Render flag\n */\nconst enum Flag {\n  TEASER = 1,                          /* Render teaser */\n  PARENT = 2                           /* Render as parent */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper function\n * ------------------------------------------------------------------------- */\n\n/**\n * Render a search document\n *\n * @param document - Search document\n * @param flag - Render flags\n *\n * @returns Element\n */\nfunction renderSearchDocument(\n  document: SearchItem, flag: Flag\n): HTMLElement {\n  const parent = flag & Flag.PARENT\n  const teaser = flag & Flag.TEASER\n\n  /* Render missing query terms */\n  const missing = Object.keys(document.terms)\n    .filter(key => !document.terms[key])\n    .reduce<ComponentChild[]>((list, key) => [\n      ...list, <del>{key}</del>, \" \"\n    ], [])\n    .slice(0, -1)\n\n  /* Assemble query string for highlighting */\n  const config = configuration()\n  const url = new URL(document.location, config.base)\n  if (feature(\"search.highlight\"))\n    url.searchParams.set(\"h\", Object.entries(document.terms)\n      .filter(([, match]) => match)\n      .reduce((highlight, [value]) => `${highlight} ${value}`.trim(), \"\")\n    )\n\n  /* Render article or section, depending on flags */\n  const { tags } = configuration()\n  return (\n    <a href={`${url}`} class=\"md-search-result__link\" tabIndex={-1}>\n      <article\n        class=\"md-search-result__article md-typeset\"\n        data-md-score={document.score.toFixed(2)}\n      >\n        {parent > 0 && <div class=\"md-search-result__icon md-icon\"></div>}\n        {parent > 0 && <h1>{document.title}</h1>}\n        {parent <= 0 && <h2>{document.title}</h2>}\n        {teaser > 0 && document.text.length > 0 &&\n          document.text\n        }\n        {document.tags && document.tags.map(tag => {\n          const type = tags\n            ? tag in tags\n              ? `md-tag-icon md-tag--${tags[tag]}`\n              : \"md-tag-icon\"\n            : \"\"\n          return (\n            <span class={`md-tag ${type}`}>{tag}</span>\n          )\n        })}\n        {teaser > 0 && missing.length > 0 &&\n          <p class=\"md-search-result__terms\">\n            {translation(\"search.result.term.missing\")}: {...missing}\n          </p>\n        }\n      </article>\n    </a>\n  )\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Render a search result\n *\n * @param result - Search result\n *\n * @returns Element\n */\nexport function renderSearchResultItem(\n  result: SearchItem[]\n): HTMLElement {\n  const threshold = result[0].score\n  const docs = [...result]\n\n  const config = configuration()\n\n  /* Find and extract parent article */\n  const parent = docs.findIndex(doc => {\n    const l = `${new URL(doc.location, config.base)}` // @todo hacky\n    return !l.includes(\"#\")\n  })\n  const [article] = docs.splice(parent, 1)\n\n  /* Determine last index above threshold */\n  let index = docs.findIndex(doc => doc.score < threshold)\n  if (index === -1)\n    index = docs.length\n\n  /* Partition sections */\n  const best = docs.slice(0, index)\n  const more = docs.slice(index)\n\n  /* Render children */\n  const children = [\n    renderSearchDocument(article, Flag.PARENT | +(!parent && index === 0)),\n    ...best.map(section => renderSearchDocument(section, Flag.TEASER)),\n    ...more.length ? [\n      <details class=\"md-search-result__more\">\n        <summary tabIndex={-1}>\n          <div>\n            {more.length > 0 && more.length === 1\n              ? translation(\"search.result.more.one\")\n              : translation(\"search.result.more.other\", more.length)\n            }\n          </div>\n        </summary>\n        {...more.map(section => renderSearchDocument(section, Flag.TEASER))}\n      </details>\n    ] : []\n  ]\n\n  /* Render search result */\n  return (\n    <li class=\"md-search-result__item\">\n      {children}\n    </li>\n  )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { SourceFacts } from \"~/components\"\nimport { h, round } from \"~/utilities\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Render repository facts\n *\n * @param facts - Repository facts\n *\n * @returns Element\n */\nexport function renderSourceFacts(facts: SourceFacts): HTMLElement {\n  return (\n    <ul class=\"md-source__facts\">\n      {Object.entries(facts).map(([key, value]) => (\n        <li class={`md-source__fact md-source__fact--${key}`}>\n          {typeof value === \"number\" ? round(value) : value}\n        </li>\n      ))}\n    </ul>\n  )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { h } from \"~/utilities\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Tabbed control type\n */\ntype TabbedControlType =\n  | \"prev\"\n  | \"next\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Render control for content tabs\n *\n * @param type - Control type\n *\n * @returns Element\n */\nexport function renderTabbedControl(\n  type: TabbedControlType\n): HTMLElement {\n  const classes = `tabbed-control tabbed-control--${type}`\n  return (\n    <div class={classes} hidden>\n      <button class=\"tabbed-button\" tabIndex={-1} aria-hidden=\"true\"></button>\n    </div>\n  )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { h } from \"~/utilities\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Render a table inside a wrapper to improve scrolling on mobile\n *\n * @param table - Table element\n *\n * @returns Element\n */\nexport function renderTable(table: HTMLElement): HTMLElement {\n  return (\n    <div class=\"md-typeset__scrollwrap\">\n      <div class=\"md-typeset__table\">\n        {table}\n      </div>\n    </div>\n  )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { configuration, translation } from \"~/_\"\nimport { h } from \"~/utilities\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Version\n */\nexport interface Version {\n  version: string                      /* Version identifier */\n  title: string                        /* Version title */\n  aliases: string[]                    /* Version aliases */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Render a version\n *\n * @param version - Version\n *\n * @returns Element\n */\nfunction renderVersion(version: Version): HTMLElement {\n  const config = configuration()\n\n  /* Ensure trailing slash - see https://bit.ly/3rL5u3f */\n  const url = new URL(`../${version.version}/`, config.base)\n  return (\n    <li class=\"md-version__item\">\n      <a href={`${url}`} class=\"md-version__link\">\n        {version.title}\n      </a>\n    </li>\n  )\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Render a version selector\n *\n * @param versions - Versions\n * @param active - Active version\n *\n * @returns Element\n */\nexport function renderVersionSelector(\n  versions: Version[], active: Version\n): HTMLElement {\n  return (\n    <div class=\"md-version\">\n      <button\n        class=\"md-version__current\"\n        aria-label={translation(\"select.version\")}\n      >\n        {active.title}\n      </button>\n      <ul class=\"md-version__list\">\n        {versions.map(renderVersion)}\n      </ul>\n    </div>\n  )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  EMPTY,\n  Observable,\n  Subject,\n  animationFrameScheduler,\n  asyncScheduler,\n  auditTime,\n  combineLatest,\n  debounceTime,\n  defer,\n  distinctUntilChanged,\n  filter,\n  finalize,\n  map,\n  merge,\n  of,\n  subscribeOn,\n  tap,\n  throttleTime\n} from \"rxjs\"\n\nimport {\n  ElementOffset,\n  getElement,\n  getElementContainer,\n  getElementOffset,\n  getElementSize,\n  watchElementContentOffset,\n  watchElementFocus,\n  watchElementHover\n} from \"~/browser\"\nimport { renderTooltip } from \"~/templates\"\n\nimport { Component } from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Tooltip\n */\nexport interface Tooltip {\n  active: boolean                      /* Tooltip is active */\n  offset: ElementOffset                /* Tooltip offset */\n}\n\n/* ----------------------------------------------------------------------------\n * Data\n * ------------------------------------------------------------------------- */\n\n/**\n * Global sequence number for tooltips\n */\nlet sequence = 0\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch tooltip\n *\n * This function will append the tooltip temporarily to compute its width,\n * which is necessary for correct centering, and then removing it again.\n *\n * @param el - Tooltip element\n * @param host - Host element\n *\n * @returns Tooltip observable\n */\nexport function watchTooltip(\n  el: HTMLElement, host: HTMLElement\n): Observable<Tooltip> {\n  document.body.append(el)\n\n  /* Compute width and remove tooltip immediately */\n  const { width } = getElementSize(el)\n  el.style.setProperty(\"--md-tooltip-width\", `${width}px`)\n  el.remove()\n\n  /* Retrieve and watch containing element */\n  const container = getElementContainer(host)\n  const scroll$ =\n    typeof container !== \"undefined\"\n      ? watchElementContentOffset(container)\n      : of({ x: 0, y: 0 })\n\n  /* Compute tooltip visibility */\n  const active$ = merge(\n    watchElementFocus(host),\n    watchElementHover(host)\n  )\n    .pipe(\n      distinctUntilChanged()\n    )\n\n  /* Compute tooltip offset */\n  return combineLatest([active$, scroll$])\n    .pipe(\n      map(([active, scroll]) => {\n        let { x, y } = getElementOffset(host)\n        const size = getElementSize(host)\n\n        /**\n         * Experimental: fix handling of tables - see https://bit.ly/3TQEj5O\n         *\n         * If this proves to be a viable fix, we should refactor tooltip\n         * positioning and somehow streamline the current process. This might\n         * also fix positioning for annotations inside tables, which is another\n         * limitation.\n         */\n        const table = host.closest(\"table\")\n        if (table && host.parentElement) {\n          x += table.offsetLeft + host.parentElement.offsetLeft\n          y += table.offsetTop  + host.parentElement.offsetTop\n        }\n        return {\n          active,\n          offset: {\n            x: x - scroll.x + size.width  / 2 - width / 2,\n            y: y - scroll.y + size.height + 8\n          }\n        }\n      })\n    )\n}\n\n/**\n * Mount tooltip\n *\n * @param el - Host element\n *\n * @returns Tooltip component observable\n */\nexport function mountTooltip(\n  el: HTMLElement\n): Observable<Component<Tooltip>> {\n  const title = el.title\n  if (!title.length)\n    return EMPTY\n\n  /* Render tooltip and set title from host element */\n  const id = `__tooltip_${sequence++}`\n  const tooltip = renderTooltip(id, \"inline\")\n  const typeset = getElement(\".md-typeset\", tooltip)\n  typeset.innerHTML = title\n\n  /* Mount component on subscription */\n  return defer(() => {\n    const push$ = new Subject<Tooltip>()\n    push$.subscribe({\n\n      /* Handle emission */\n      next({ offset }) {\n        tooltip.style.setProperty(\"--md-tooltip-x\", `${offset.x}px`)\n        tooltip.style.setProperty(\"--md-tooltip-y\", `${offset.y}px`)\n      },\n\n      /* Handle complete */\n      complete() {\n        tooltip.style.removeProperty(\"--md-tooltip-x\")\n        tooltip.style.removeProperty(\"--md-tooltip-y\")\n      }\n    })\n\n    /* Toggle tooltip presence to mitigate empty lines when copying */\n    merge(\n      push$.pipe(filter(({ active }) => active)),\n      push$.pipe(debounceTime(250), filter(({ active }) => !active))\n    )\n      .subscribe({\n\n        /* Handle emission */\n        next({ active }) {\n          if (active) {\n            el.insertAdjacentElement(\"afterend\", tooltip)\n            el.setAttribute(\"aria-describedby\", id)\n            el.removeAttribute(\"title\")\n          } else {\n            tooltip.remove()\n            el.removeAttribute(\"aria-describedby\")\n            el.setAttribute(\"title\", title)\n          }\n        },\n\n        /* Handle complete */\n        complete() {\n          tooltip.remove()\n          el.removeAttribute(\"aria-describedby\")\n          el.setAttribute(\"title\", title)\n        }\n      })\n\n    /* Toggle tooltip visibility */\n    push$\n      .pipe(\n        auditTime(16, animationFrameScheduler)\n      )\n        .subscribe(({ active }) => {\n          tooltip.classList.toggle(\"md-tooltip--active\", active)\n        })\n\n    // @todo - refactor positioning together with annotations \u2013 there are\n    // several things that overlap and are identical in handling\n\n    /* Track relative origin of tooltip */\n    push$\n      .pipe(\n        throttleTime(125, animationFrameScheduler),\n        filter(() => !!el.offsetParent),\n        map(() => el.offsetParent!.getBoundingClientRect()),\n        map(({ x }) => x)\n      )\n      .subscribe({\n\n        /* Handle emission */\n        next(origin) {\n          if (origin)\n            tooltip.style.setProperty(\"--md-tooltip-0\", `${-origin}px`)\n          else\n            tooltip.style.removeProperty(\"--md-tooltip-0\")\n        },\n\n        /* Handle complete */\n        complete() {\n          tooltip.style.removeProperty(\"--md-tooltip-0\")\n        }\n      })\n\n    /* Create and return component */\n    return watchTooltip(tooltip, el)\n      .pipe(\n        tap(state => push$.next(state)),\n        finalize(() => push$.complete()),\n        map(state => ({ ref: el, ...state }))\n      )\n  })\n    .pipe(\n      subscribeOn(asyncScheduler)\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  animationFrameScheduler,\n  auditTime,\n  combineLatest,\n  debounceTime,\n  defer,\n  delay,\n  endWith,\n  filter,\n  finalize,\n  fromEvent,\n  ignoreElements,\n  map,\n  merge,\n  switchMap,\n  take,\n  takeUntil,\n  tap,\n  throttleTime,\n  withLatestFrom\n} from \"rxjs\"\n\nimport {\n  ElementOffset,\n  getActiveElement,\n  getElementSize,\n  watchElementContentOffset,\n  watchElementFocus,\n  watchElementOffset,\n  watchElementVisibility\n} from \"~/browser\"\n\nimport { Component } from \"../../../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Annotation\n */\nexport interface Annotation {\n  active: boolean                      /* Annotation is active */\n  offset: ElementOffset                /* Annotation offset */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  target$: Observable<HTMLElement>     /* Location target observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch annotation\n *\n * @param el - Annotation element\n * @param container - Containing element\n *\n * @returns Annotation observable\n */\nexport function watchAnnotation(\n  el: HTMLElement, container: HTMLElement\n): Observable<Annotation> {\n  const offset$ = defer(() => combineLatest([\n    watchElementOffset(el),\n    watchElementContentOffset(container)\n  ]))\n    .pipe(\n      map(([{ x, y }, scroll]): ElementOffset => {\n        const { width, height } = getElementSize(el)\n        return ({\n          x: x - scroll.x + width  / 2,\n          y: y - scroll.y + height / 2\n        })\n      })\n    )\n\n  /* Actively watch annotation on focus */\n  return watchElementFocus(el)\n    .pipe(\n      switchMap(active => offset$\n        .pipe(\n          map(offset => ({ active, offset })),\n          take(+!active || Infinity)\n        )\n      )\n    )\n}\n\n/**\n * Mount annotation\n *\n * @param el - Annotation element\n * @param container - Containing element\n * @param options - Options\n *\n * @returns Annotation component observable\n */\nexport function mountAnnotation(\n  el: HTMLElement, container: HTMLElement, { target$ }: MountOptions\n): Observable<Component<Annotation>> {\n  const [tooltip, index] = Array.from(el.children)\n\n  /* Mount component on subscription */\n  return defer(() => {\n    const push$ = new Subject<Annotation>()\n    const done$ = push$.pipe(ignoreElements(), endWith(true))\n    push$.subscribe({\n\n      /* Handle emission */\n      next({ offset }) {\n        el.style.setProperty(\"--md-tooltip-x\", `${offset.x}px`)\n        el.style.setProperty(\"--md-tooltip-y\", `${offset.y}px`)\n      },\n\n      /* Handle complete */\n      complete() {\n        el.style.removeProperty(\"--md-tooltip-x\")\n        el.style.removeProperty(\"--md-tooltip-y\")\n      }\n    })\n\n    /* Start animation only when annotation is visible */\n    watchElementVisibility(el)\n      .pipe(\n        takeUntil(done$)\n      )\n        .subscribe(visible => {\n          el.toggleAttribute(\"data-md-visible\", visible)\n        })\n\n    /* Toggle tooltip presence to mitigate empty lines when copying */\n    merge(\n      push$.pipe(filter(({ active }) => active)),\n      push$.pipe(debounceTime(250), filter(({ active }) => !active))\n    )\n      .subscribe({\n\n        /* Handle emission */\n        next({ active }) {\n          if (active)\n            el.prepend(tooltip)\n          else\n            tooltip.remove()\n        },\n\n        /* Handle complete */\n        complete() {\n          el.prepend(tooltip)\n        }\n      })\n\n    /* Toggle tooltip visibility */\n    push$\n      .pipe(\n        auditTime(16, animationFrameScheduler)\n      )\n        .subscribe(({ active }) => {\n          tooltip.classList.toggle(\"md-tooltip--active\", active)\n        })\n\n    /* Track relative origin of tooltip */\n    push$\n      .pipe(\n        throttleTime(125, animationFrameScheduler),\n        filter(() => !!el.offsetParent),\n        map(() => el.offsetParent!.getBoundingClientRect()),\n        map(({ x }) => x)\n      )\n        .subscribe({\n\n          /* Handle emission */\n          next(origin) {\n            if (origin)\n              el.style.setProperty(\"--md-tooltip-0\", `${-origin}px`)\n            else\n              el.style.removeProperty(\"--md-tooltip-0\")\n          },\n\n          /* Handle complete */\n          complete() {\n            el.style.removeProperty(\"--md-tooltip-0\")\n          }\n        })\n\n    /* Allow to copy link without scrolling to anchor */\n    fromEvent<MouseEvent>(index, \"click\")\n      .pipe(\n        takeUntil(done$),\n        filter(ev => !(ev.metaKey || ev.ctrlKey))\n      )\n        .subscribe(ev => {\n          ev.stopPropagation()\n          ev.preventDefault()\n        })\n\n    /* Allow to open link in new tab or blur on close */\n    fromEvent<MouseEvent>(index, \"mousedown\")\n      .pipe(\n        takeUntil(done$),\n        withLatestFrom(push$)\n      )\n        .subscribe(([ev, { active }]) => {\n\n          /* Open in new tab */\n          if (ev.button !== 0 || ev.metaKey || ev.ctrlKey) {\n            ev.preventDefault()\n\n          /* Close annotation */\n          } else if (active) {\n            ev.preventDefault()\n\n            /* Focus parent annotation, if any */\n            const parent = el.parentElement!.closest(\".md-annotation\")\n            if (parent instanceof HTMLElement)\n              parent.focus()\n            else\n              getActiveElement()?.blur()\n          }\n        })\n\n    /* Open and focus annotation on location target */\n    target$\n      .pipe(\n        takeUntil(done$),\n        filter(target => target === tooltip),\n        delay(125)\n      )\n        .subscribe(() => el.focus())\n\n    /* Create and return component */\n    return watchAnnotation(el, container)\n      .pipe(\n        tap(state => push$.next(state)),\n        finalize(() => push$.complete()),\n        map(state => ({ ref: el, ...state }))\n      )\n  })\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  EMPTY,\n  Observable,\n  Subject,\n  defer,\n  endWith,\n  finalize,\n  ignoreElements,\n  merge,\n  share,\n  takeUntil\n} from \"rxjs\"\n\nimport {\n  getElement,\n  getElements,\n  getOptionalElement\n} from \"~/browser\"\nimport { renderAnnotation } from \"~/templates\"\n\nimport { Component } from \"../../../_\"\nimport {\n  Annotation,\n  mountAnnotation\n} from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  target$: Observable<HTMLElement>     /* Location target observable */\n  print$: Observable<boolean>          /* Media print observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Find all annotation hosts in the containing element\n *\n * @param container - Containing element\n *\n * @returns Annotation hosts\n */\nfunction findHosts(container: HTMLElement): HTMLElement[] {\n  return container.tagName === \"CODE\"\n    ? getElements(\".c, .c1, .cm\", container)\n    : [container]\n}\n\n/**\n * Find all annotation markers in the containing element\n *\n * @param container - Containing element\n *\n * @returns Annotation markers\n */\nfunction findMarkers(container: HTMLElement): Text[] {\n  const markers: Text[] = []\n  for (const el of findHosts(container)) {\n    const nodes: Text[] = []\n\n    /* Find all text nodes in current element */\n    const it = document.createNodeIterator(el, NodeFilter.SHOW_TEXT)\n    for (let node = it.nextNode(); node; node = it.nextNode())\n      nodes.push(node as Text)\n\n    /* Find all markers in each text node */\n    for (let text of nodes) {\n      let match: RegExpExecArray | null\n\n      /* Split text at marker and add to list */\n      while ((match = /(\\(\\d+\\))(!)?/.exec(text.textContent!))) {\n        const [, id, force] = match\n        if (typeof force === \"undefined\") {\n          const marker = text.splitText(match.index)\n          text = marker.splitText(id.length)\n          markers.push(marker)\n\n        /* Replace entire text with marker */\n        } else {\n          text.textContent = id\n          markers.push(text)\n          break\n        }\n      }\n    }\n  }\n  return markers\n}\n\n/**\n * Swap the child nodes of two elements\n *\n * @param source - Source element\n * @param target - Target element\n */\nfunction swap(source: HTMLElement, target: HTMLElement): void {\n  target.append(...Array.from(source.childNodes))\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount annotation list\n *\n * This function analyzes the containing code block and checks for markers\n * referring to elements in the given annotation list. If no markers are found,\n * the list is left untouched. Otherwise, list elements are rendered as\n * annotations inside the code block.\n *\n * @param el - Annotation list element\n * @param container - Containing element\n * @param options - Options\n *\n * @returns Annotation component observable\n */\nexport function mountAnnotationList(\n  el: HTMLElement, container: HTMLElement, { target$, print$ }: MountOptions\n): Observable<Component<Annotation>> {\n\n  /* Compute prefix for tooltip anchors */\n  const parent = container.closest(\"[id]\")\n  const prefix = parent?.id\n\n  /* Find and replace all markers with empty annotations */\n  const annotations = new Map<string, HTMLElement>()\n  for (const marker of findMarkers(container)) {\n    const [, id] = marker.textContent!.match(/\\((\\d+)\\)/)!\n    if (getOptionalElement(`:scope > li:nth-child(${id})`, el)) {\n      annotations.set(id, renderAnnotation(id, prefix))\n      marker.replaceWith(annotations.get(id)!)\n    }\n  }\n\n  /* Keep list if there are no annotations to render */\n  if (annotations.size === 0)\n    return EMPTY\n\n  /* Mount component on subscription */\n  return defer(() => {\n    const push$ = new Subject()\n    const done$ = push$.pipe(ignoreElements(), endWith(true))\n\n    /* Retrieve container pairs for swapping */\n    const pairs: [HTMLElement, HTMLElement][] = []\n    for (const [id, annotation] of annotations)\n      pairs.push([\n        getElement(\".md-typeset\", annotation),\n        getElement(`:scope > li:nth-child(${id})`, el)\n      ])\n\n    /* Handle print mode - see https://bit.ly/3rgPdpt */\n    print$.pipe(takeUntil(done$))\n      .subscribe(active => {\n        el.hidden = !active\n\n        /* Add class to discern list element */\n        el.classList.toggle(\"md-annotation-list\", active)\n\n        /* Show annotations in code block or list (print) */\n        for (const [inner, child] of pairs)\n          if (!active)\n            swap(child, inner)\n          else\n            swap(inner, child)\n      })\n\n    /* Create and return component */\n    return merge(...[...annotations]\n      .map(([, annotation]) => (\n        mountAnnotation(annotation, container, { target$ })\n      ))\n    )\n      .pipe(\n        finalize(() => push$.complete()),\n        share()\n      )\n  })\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { EMPTY, Observable, defer } from \"rxjs\"\n\nimport { Component } from \"../../../_\"\nimport { Annotation } from \"../_\"\nimport { mountAnnotationList } from \"../list\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  target$: Observable<HTMLElement>     /* Location target observable */\n  print$: Observable<boolean>          /* Media print observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Find list element directly following a block\n *\n * @param el - Annotation block element\n *\n * @returns List element or nothing\n */\nfunction findList(el: HTMLElement): HTMLElement | undefined {\n  if (el.nextElementSibling) {\n    const sibling = el.nextElementSibling as HTMLElement\n    if (sibling.tagName === \"OL\")\n      return sibling\n\n    /* Skip empty paragraphs - see https://bit.ly/3r4ZJ2O */\n    else if (sibling.tagName === \"P\" && !sibling.children.length)\n      return findList(sibling)\n  }\n\n  /* Everything else */\n  return undefined\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount annotation block\n *\n * @param el - Annotation block element\n * @param options - Options\n *\n * @returns Annotation component observable\n */\nexport function mountAnnotationBlock(\n  el: HTMLElement, options: MountOptions\n): Observable<Component<Annotation>> {\n  return defer(() => {\n    const list = findList(el)\n    return typeof list !== \"undefined\"\n      ? mountAnnotationList(list, el, options)\n      : EMPTY\n  })\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport ClipboardJS from \"clipboard\"\nimport {\n  EMPTY,\n  Observable,\n  Subject,\n  defer,\n  distinctUntilChanged,\n  distinctUntilKeyChanged,\n  filter,\n  finalize,\n  map,\n  mergeWith,\n  switchMap,\n  take,\n  takeLast,\n  takeUntil,\n  tap\n} from \"rxjs\"\n\nimport { feature } from \"~/_\"\nimport {\n  getElementContentSize,\n  watchElementSize,\n  watchElementVisibility\n} from \"~/browser\"\nimport { renderClipboardButton } from \"~/templates\"\n\nimport { Component } from \"../../../_\"\nimport {\n  Tooltip,\n  mountTooltip\n} from \"../../../tooltip\"\nimport {\n  Annotation,\n  mountAnnotationList\n} from \"../../annotation\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Code block overflow\n */\nexport interface Overflow {\n  scrollable: boolean                  /* Code block overflows */\n}\n\n/**\n * Code block\n */\nexport type CodeBlock =\n  | Overflow\n  | Annotation\n  | Tooltip\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  target$: Observable<HTMLElement>     /* Location target observable */\n  print$: Observable<boolean>          /* Media print observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Data\n * ------------------------------------------------------------------------- */\n\n/**\n * Global sequence number for code blocks\n */\nlet sequence = 0\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Find candidate list element directly following a code block\n *\n * @param el - Code block element\n *\n * @returns List element or nothing\n */\nfunction findCandidateList(el: HTMLElement): HTMLElement | undefined {\n  if (el.nextElementSibling) {\n    const sibling = el.nextElementSibling as HTMLElement\n    if (sibling.tagName === \"OL\")\n      return sibling\n\n    /* Skip empty paragraphs - see https://bit.ly/3r4ZJ2O */\n    else if (sibling.tagName === \"P\" && !sibling.children.length)\n      return findCandidateList(sibling)\n  }\n\n  /* Everything else */\n  return undefined\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch code block\n *\n * This function monitors size changes of the viewport, as well as switches of\n * content tabs with embedded code blocks, as both may trigger overflow.\n *\n * @param el - Code block element\n *\n * @returns Code block observable\n */\nexport function watchCodeBlock(\n  el: HTMLElement\n): Observable<Overflow> {\n  return watchElementSize(el)\n    .pipe(\n      map(({ width }) => {\n        const content = getElementContentSize(el)\n        return {\n          scrollable: content.width > width\n        }\n      }),\n      distinctUntilKeyChanged(\"scrollable\")\n    )\n}\n\n/**\n * Mount code block\n *\n * This function ensures that an overflowing code block is focusable through\n * keyboard, so it can be scrolled without a mouse to improve on accessibility.\n * Furthermore, if code annotations are enabled, they are mounted if and only\n * if the code block is currently visible, e.g., not in a hidden content tab.\n *\n * Note that code blocks may be mounted eagerly or lazily. If they're mounted\n * lazily (on first visibility), code annotation anchor links will not work,\n * as they are evaluated on initial page load, and code annotations in general\n * might feel a little bumpier.\n *\n * @param el - Code block element\n * @param options - Options\n *\n * @returns Code block and annotation component observable\n */\nexport function mountCodeBlock(\n  el: HTMLElement, options: MountOptions\n): Observable<Component<CodeBlock>> {\n  const { matches: hover } = matchMedia(\"(hover)\")\n\n  /* Defer mounting of code block - see https://bit.ly/3vHVoVD */\n  const factory$ = defer(() => {\n    const push$ = new Subject<Overflow>()\n    const done$ = push$.pipe(takeLast(1))\n    push$.subscribe(({ scrollable }) => {\n      if (scrollable && hover)\n        el.setAttribute(\"tabindex\", \"0\")\n      else\n        el.removeAttribute(\"tabindex\")\n    })\n\n    /* Render button for Clipboard.js integration */\n    const content$: Array<Observable<Component<CodeBlock>>> = []\n    if (ClipboardJS.isSupported()) {\n      if (el.closest(\".copy\") || (\n        feature(\"content.code.copy\") && !el.closest(\".no-copy\")\n      )) {\n        const parent = el.closest(\"pre\")!\n        parent.id = `__code_${sequence++}`\n\n        /* Mount tooltip, if enabled */\n        const button = renderClipboardButton(parent.id)\n        parent.insertBefore(button, el)\n        if (feature(\"content.tooltips\"))\n          content$.push(mountTooltip(button))\n      }\n    }\n\n    /* Handle code annotations */\n    const container = el.closest(\".highlight\")\n    if (container instanceof HTMLElement) {\n      const list = findCandidateList(container)\n\n      /* Mount code annotations, if enabled */\n      if (typeof list !== \"undefined\" && (\n        container.classList.contains(\"annotate\") ||\n        feature(\"content.code.annotate\")\n      )) {\n        const annotations$ = mountAnnotationList(list, el, options)\n        content$.push(\n          watchElementSize(container)\n            .pipe(\n              takeUntil(done$),\n              map(({ width, height }) => width && height),\n              distinctUntilChanged(),\n              switchMap(active => active ? annotations$ : EMPTY)\n            )\n        )\n      }\n    }\n\n    /* Create and return component */\n    return watchCodeBlock(el)\n      .pipe(\n        tap(state => push$.next(state)),\n        finalize(() => push$.complete()),\n        map(state => ({ ref: el, ...state })),\n        mergeWith(...content$)\n      )\n  })\n\n  /* Mount code block lazily */\n  if (feature(\"content.lazy\"))\n    return watchElementVisibility(el)\n      .pipe(\n        filter(visible => visible),\n        take(1),\n        switchMap(() => factory$)\n      )\n\n  /* Mount code block */\n  return factory$\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  defer,\n  filter,\n  finalize,\n  map,\n  merge,\n  tap\n} from \"rxjs\"\n\nimport { Component } from \"../../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Details\n */\nexport interface Details {\n  action: \"open\" | \"close\"             /* Details state */\n  reveal?: boolean                     /* Details is revealed */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  target$: Observable<HTMLElement>     /* Location target observable */\n  print$: Observable<boolean>          /* Media print observable */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  target$: Observable<HTMLElement>     /* Location target observable */\n  print$: Observable<boolean>          /* Media print observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch details\n *\n * @param el - Details element\n * @param options - Options\n *\n * @returns Details observable\n */\nexport function watchDetails(\n  el: HTMLDetailsElement, { target$, print$ }: WatchOptions\n): Observable<Details> {\n  let open = true\n  return merge(\n\n    /* Open and focus details on location target */\n    target$\n      .pipe(\n        map(target => target.closest(\"details:not([open])\")!),\n        filter(details => el === details),\n        map(() => ({\n          action: \"open\", reveal: true\n        }) as Details)\n      ),\n\n    /* Open details on print and close afterwards */\n    print$\n      .pipe(\n        filter(active => active || !open),\n        tap(() => open = el.open),\n        map(active => ({\n          action: active ? \"open\" : \"close\"\n        }) as Details)\n      )\n  )\n}\n\n/**\n * Mount details\n *\n * This function ensures that `details` tags are opened on anchor jumps and\n * prior to printing, so the whole content of the page is visible.\n *\n * @param el - Details element\n * @param options - Options\n *\n * @returns Details component observable\n */\nexport function mountDetails(\n  el: HTMLDetailsElement, options: MountOptions\n): Observable<Component<Details>> {\n  return defer(() => {\n    const push$ = new Subject<Details>()\n    push$.subscribe(({ action, reveal }) => {\n      el.toggleAttribute(\"open\", action === \"open\")\n      if (reveal)\n        el.scrollIntoView()\n    })\n\n    /* Create and return component */\n    return watchDetails(el, options)\n      .pipe(\n        tap(state => push$.next(state)),\n        finalize(() => push$.complete()),\n        map(state => ({ ref: el, ...state }))\n      )\n  })\n}\n", ".node circle,.node ellipse,.node path,.node polygon,.node rect{fill:var(--md-mermaid-node-bg-color);stroke:var(--md-mermaid-node-fg-color)}marker{fill:var(--md-mermaid-edge-color)!important}.edgeLabel .label rect{fill:#0000}.label{color:var(--md-mermaid-label-fg-color);font-family:var(--md-mermaid-font-family)}.label foreignObject{line-height:normal;overflow:visible}.label div .edgeLabel{color:var(--md-mermaid-label-fg-color)}.edgeLabel,.edgeLabel rect,.label div .edgeLabel{background-color:var(--md-mermaid-label-bg-color)}.edgeLabel,.edgeLabel rect{fill:var(--md-mermaid-label-bg-color);color:var(--md-mermaid-edge-color)}.edgePath .path,.flowchart-link{stroke:var(--md-mermaid-edge-color);stroke-width:.05rem}.edgePath .arrowheadPath{fill:var(--md-mermaid-edge-color);stroke:none}.cluster rect{fill:var(--md-default-fg-color--lightest);stroke:var(--md-default-fg-color--lighter)}.cluster span{color:var(--md-mermaid-label-fg-color);font-family:var(--md-mermaid-font-family)}g #flowchart-circleEnd,g #flowchart-circleStart,g #flowchart-crossEnd,g #flowchart-crossStart,g #flowchart-pointEnd,g #flowchart-pointStart{stroke:none}g.classGroup line,g.classGroup rect{fill:var(--md-mermaid-node-bg-color);stroke:var(--md-mermaid-node-fg-color)}g.classGroup text{fill:var(--md-mermaid-label-fg-color);font-family:var(--md-mermaid-font-family)}.classLabel .box{fill:var(--md-mermaid-label-bg-color);background-color:var(--md-mermaid-label-bg-color);opacity:1}.classLabel .label{fill:var(--md-mermaid-label-fg-color);font-family:var(--md-mermaid-font-family)}.node .divider{stroke:var(--md-mermaid-node-fg-color)}.relation{stroke:var(--md-mermaid-edge-color)}.cardinality{fill:var(--md-mermaid-label-fg-color);font-family:var(--md-mermaid-font-family)}.cardinality text{fill:inherit!important}defs #classDiagram-compositionEnd,defs #classDiagram-compositionStart,defs #classDiagram-dependencyEnd,defs #classDiagram-dependencyStart,defs #classDiagram-extensionEnd,defs #classDiagram-extensionStart{fill:var(--md-mermaid-edge-color)!important;stroke:var(--md-mermaid-edge-color)!important}defs #classDiagram-aggregationEnd,defs #classDiagram-aggregationStart{fill:var(--md-mermaid-label-bg-color)!important;stroke:var(--md-mermaid-edge-color)!important}g.stateGroup rect{fill:var(--md-mermaid-node-bg-color);stroke:var(--md-mermaid-node-fg-color)}g.stateGroup .state-title{fill:var(--md-mermaid-label-fg-color)!important;font-family:var(--md-mermaid-font-family)}g.stateGroup .composit{fill:var(--md-mermaid-label-bg-color)}.nodeLabel,.nodeLabel p{color:var(--md-mermaid-label-fg-color);font-family:var(--md-mermaid-font-family)}.node circle.state-end,.node circle.state-start,.start-state{fill:var(--md-mermaid-edge-color);stroke:none}.end-state-inner,.end-state-outer{fill:var(--md-mermaid-edge-color)}.end-state-inner,.node circle.state-end{stroke:var(--md-mermaid-label-bg-color)}.transition{stroke:var(--md-mermaid-edge-color)}[id^=state-fork] rect,[id^=state-join] rect{fill:var(--md-mermaid-edge-color)!important;stroke:none!important}.statediagram-cluster.statediagram-cluster .inner{fill:var(--md-default-bg-color)}.statediagram-cluster rect{fill:var(--md-mermaid-node-bg-color);stroke:var(--md-mermaid-node-fg-color)}.statediagram-state rect.divider{fill:var(--md-default-fg-color--lightest);stroke:var(--md-default-fg-color--lighter)}defs #statediagram-barbEnd{stroke:var(--md-mermaid-edge-color)}.attributeBoxEven,.attributeBoxOdd{fill:var(--md-mermaid-node-bg-color);stroke:var(--md-mermaid-node-fg-color)}.entityBox{fill:var(--md-mermaid-label-bg-color);stroke:var(--md-mermaid-node-fg-color)}.entityLabel{fill:var(--md-mermaid-label-fg-color);font-family:var(--md-mermaid-font-family)}.relationshipLabelBox{fill:var(--md-mermaid-label-bg-color);fill-opacity:1;background-color:var(--md-mermaid-label-bg-color);opacity:1}.relationshipLabel{fill:var(--md-mermaid-label-fg-color)}.relationshipLine{stroke:var(--md-mermaid-edge-color)}defs #ONE_OR_MORE_END *,defs #ONE_OR_MORE_START *,defs #ONLY_ONE_END *,defs #ONLY_ONE_START *,defs #ZERO_OR_MORE_END *,defs #ZERO_OR_MORE_START *,defs #ZERO_OR_ONE_END *,defs #ZERO_OR_ONE_START *{stroke:var(--md-mermaid-edge-color)!important}defs #ZERO_OR_MORE_END circle,defs #ZERO_OR_MORE_START circle{fill:var(--md-mermaid-label-bg-color)}.actor{fill:var(--md-mermaid-sequence-actor-bg-color);stroke:var(--md-mermaid-sequence-actor-border-color)}text.actor>tspan{fill:var(--md-mermaid-sequence-actor-fg-color);font-family:var(--md-mermaid-font-family)}line{stroke:var(--md-mermaid-sequence-actor-line-color)}.actor-man circle,.actor-man line{fill:var(--md-mermaid-sequence-actorman-bg-color);stroke:var(--md-mermaid-sequence-actorman-line-color)}.messageLine0,.messageLine1{stroke:var(--md-mermaid-sequence-message-line-color)}.note{fill:var(--md-mermaid-sequence-note-bg-color);stroke:var(--md-mermaid-sequence-note-border-color)}.loopText,.loopText>tspan,.messageText,.noteText>tspan{stroke:none;font-family:var(--md-mermaid-font-family)!important}.messageText{fill:var(--md-mermaid-sequence-message-fg-color)}.loopText,.loopText>tspan{fill:var(--md-mermaid-sequence-loop-fg-color)}.noteText>tspan{fill:var(--md-mermaid-sequence-note-fg-color)}#arrowhead path{fill:var(--md-mermaid-sequence-message-line-color);stroke:none}.loopLine{fill:var(--md-mermaid-sequence-loop-bg-color);stroke:var(--md-mermaid-sequence-loop-border-color)}.labelBox{fill:var(--md-mermaid-sequence-label-bg-color);stroke:none}.labelText,.labelText>span{fill:var(--md-mermaid-sequence-label-fg-color);font-family:var(--md-mermaid-font-family)}.sequenceNumber{fill:var(--md-mermaid-sequence-number-fg-color)}rect.rect{fill:var(--md-mermaid-sequence-box-bg-color);stroke:none}rect.rect+text.text{fill:var(--md-mermaid-sequence-box-fg-color)}defs #sequencenumber{fill:var(--md-mermaid-sequence-number-bg-color)!important}", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  map,\n  of,\n  shareReplay,\n  tap\n} from \"rxjs\"\n\nimport { watchScript } from \"~/browser\"\nimport { h } from \"~/utilities\"\n\nimport { Component } from \"../../_\"\n\nimport themeCSS from \"./index.css\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Mermaid diagram\n */\nexport interface Mermaid {}\n\n/* ----------------------------------------------------------------------------\n * Data\n * ------------------------------------------------------------------------- */\n\n/**\n * Mermaid instance observable\n */\nlet mermaid$: Observable<void>\n\n/**\n * Global sequence number for diagrams\n */\nlet sequence = 0\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Fetch Mermaid script\n *\n * @returns Mermaid scripts observable\n */\nfunction fetchScripts(): Observable<void> {\n  return typeof mermaid === \"undefined\" || mermaid instanceof Element\n    ? watchScript(\"https://unpkg.com/mermaid@10.7.0/dist/mermaid.min.js\")\n    : of(undefined)\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount Mermaid diagram\n *\n * @param el - Code block element\n *\n * @returns Mermaid diagram component observable\n */\nexport function mountMermaid(\n  el: HTMLElement\n): Observable<Component<Mermaid>> {\n  el.classList.remove(\"mermaid\") // Hack: mitigate https://bit.ly/3CiN6Du\n  mermaid$ ||= fetchScripts()\n    .pipe(\n      tap(() => mermaid.initialize({\n        startOnLoad: false,\n        themeCSS,\n        sequence: {\n          actorFontSize: \"16px\", // Hack: mitigate https://bit.ly/3y0NEi3\n          messageFontSize: \"16px\",\n          noteFontSize: \"16px\"\n        }\n      })),\n      map(() => undefined),\n      shareReplay(1)\n    )\n\n  /* Render diagram */\n  mermaid$.subscribe(async () => {\n    el.classList.add(\"mermaid\") // Hack: mitigate https://bit.ly/3CiN6Du\n    const id = `__mermaid_${sequence++}`\n\n    /* Create host element to replace code block */\n    const host = h(\"div\", { class: \"mermaid\" })\n    const text = el.textContent\n\n    /* Render and inject diagram */\n    const { svg, fn } = await mermaid.render(id, text)\n\n    /* Create a shadow root and inject diagram */\n    const shadow = host.attachShadow({ mode: \"closed\" })\n    shadow.innerHTML = svg\n\n    /* Replace code block with diagram and bind functions */\n    el.replaceWith(host)\n    fn?.(shadow)\n  })\n\n  /* Create and return component */\n  return mermaid$\n    .pipe(\n      map(() => ({ ref: el }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Observable, of } from \"rxjs\"\n\nimport { renderTable } from \"~/templates\"\nimport { h } from \"~/utilities\"\n\nimport { Component } from \"../../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Data table\n */\nexport interface DataTable {}\n\n/* ----------------------------------------------------------------------------\n * Data\n * ------------------------------------------------------------------------- */\n\n/**\n * Sentinel for replacement\n */\nconst sentinel = h(\"table\")\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount data table\n *\n * This function wraps a data table in another scrollable container, so it can\n * be smoothly scrolled on smaller screen sizes and won't break the layout.\n *\n * @param el - Data table element\n *\n * @returns Data table component observable\n */\nexport function mountDataTable(\n  el: HTMLElement\n): Observable<Component<DataTable>> {\n  el.replaceWith(sentinel)\n  sentinel.replaceWith(renderTable(el))\n\n  /* Create and return component */\n  return of({ ref: el })\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  animationFrameScheduler,\n  asyncScheduler,\n  auditTime,\n  combineLatest,\n  defer,\n  endWith,\n  filter,\n  finalize,\n  fromEvent,\n  ignoreElements,\n  map,\n  merge,\n  skip,\n  startWith,\n  subscribeOn,\n  takeUntil,\n  tap,\n  withLatestFrom\n} from \"rxjs\"\n\nimport { feature } from \"~/_\"\nimport {\n  Viewport,\n  getElement,\n  getElementContentOffset,\n  getElementContentSize,\n  getElementOffset,\n  getElementSize,\n  getElements,\n  watchElementContentOffset,\n  watchElementSize\n} from \"~/browser\"\nimport { renderTabbedControl } from \"~/templates\"\nimport { h } from \"~/utilities\"\n\nimport { Component } from \"../../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Content tabs\n */\nexport interface ContentTabs {\n  active: HTMLLabelElement             /* Active tab label */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  target$: Observable<HTMLElement>     /* Location target observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch content tabs\n *\n * @param inputs - Content tabs input elements\n *\n * @returns Content tabs observable\n */\nexport function watchContentTabs(\n  inputs: HTMLInputElement[]\n): Observable<ContentTabs> {\n  const initial = inputs.find(input => input.checked) || inputs[0]\n  return merge(...inputs.map(input => fromEvent(input, \"change\")\n    .pipe(\n      map(() => getElement<HTMLLabelElement>(`label[for=\"${input.id}\"]`))\n    )\n  ))\n    .pipe(\n      startWith(getElement<HTMLLabelElement>(`label[for=\"${initial.id}\"]`)),\n      map(active => ({ active }))\n    )\n}\n\n/**\n * Mount content tabs\n *\n * @param el - Content tabs element\n * @param options - Options\n *\n * @returns Content tabs component observable\n */\nexport function mountContentTabs(\n  el: HTMLElement, { viewport$, target$ }: MountOptions\n): Observable<Component<ContentTabs>> {\n  const container = getElement(\".tabbed-labels\", el)\n  const inputs = getElements<HTMLInputElement>(\":scope > input\", el)\n\n  /* Render content tab previous button for pagination */\n  const prev = renderTabbedControl(\"prev\")\n  el.append(prev)\n\n  /* Render content tab next button for pagination */\n  const next = renderTabbedControl(\"next\")\n  el.append(next)\n\n  /* Mount component on subscription */\n  return defer(() => {\n    const push$ = new Subject<ContentTabs>()\n    const done$ = push$.pipe(ignoreElements(), endWith(true))\n    combineLatest([push$, watchElementSize(el)])\n      .pipe(\n        takeUntil(done$),\n        auditTime(1, animationFrameScheduler)\n      )\n        .subscribe({\n\n          /* Handle emission */\n          next([{ active }, size]) {\n            const offset = getElementOffset(active)\n            const { width } = getElementSize(active)\n\n            /* Set tab indicator offset and width */\n            el.style.setProperty(\"--md-indicator-x\", `${offset.x}px`)\n            el.style.setProperty(\"--md-indicator-width\", `${width}px`)\n\n            /* Scroll container to active content tab */\n            const content = getElementContentOffset(container)\n            if (\n              offset.x         < content.x              ||\n              offset.x + width > content.x + size.width\n            )\n              container.scrollTo({\n                left: Math.max(0, offset.x - 16),\n                behavior: \"smooth\"\n              })\n          },\n\n          /* Handle complete */\n          complete() {\n            el.style.removeProperty(\"--md-indicator-x\")\n            el.style.removeProperty(\"--md-indicator-width\")\n          }\n        })\n\n    /* Hide content tab buttons on borders */\n    combineLatest([\n      watchElementContentOffset(container),\n      watchElementSize(container)\n    ])\n      .pipe(\n        takeUntil(done$)\n      )\n        .subscribe(([offset, size]) => {\n          const content = getElementContentSize(container)\n          prev.hidden = offset.x < 16\n          next.hidden = offset.x > content.width - size.width - 16\n        })\n\n    /* Paginate content tab container on click */\n    merge(\n      fromEvent(prev, \"click\").pipe(map(() => -1)),\n      fromEvent(next, \"click\").pipe(map(() => +1))\n    )\n      .pipe(\n        takeUntil(done$)\n      )\n        .subscribe(direction => {\n          const { width } = getElementSize(container)\n          container.scrollBy({\n            left: width * direction,\n            behavior: \"smooth\"\n          })\n        })\n\n    /* Switch to content tab target */\n    target$\n      .pipe(\n        takeUntil(done$),\n        filter(input => inputs.includes(input as HTMLInputElement))\n      )\n        .subscribe(input => input.click())\n\n    /* Add link to each content tab label */\n    container.classList.add(\"tabbed-labels--linked\")\n    for (const input of inputs) {\n      const label = getElement<HTMLLabelElement>(`label[for=\"${input.id}\"]`)\n      label.replaceChildren(h(\"a\", {\n        href: `#${label.htmlFor}`,\n        tabIndex: -1\n      }, ...Array.from(label.childNodes)))\n\n      /* Allow to copy link without scrolling to anchor */\n      fromEvent<MouseEvent>(label.firstElementChild!, \"click\")\n        .pipe(\n          takeUntil(done$),\n          filter(ev => !(ev.metaKey || ev.ctrlKey)),\n          tap(ev => {\n            ev.preventDefault()\n            ev.stopPropagation()\n          })\n        )\n          // @todo we might need to remove the anchor link on complete\n          .subscribe(() => {\n            history.replaceState({}, \"\", `#${label.htmlFor}`)\n            label.click()\n          })\n    }\n\n    /* Set up linking of content tabs, if enabled */\n    if (feature(\"content.tabs.link\"))\n      push$.pipe(\n        skip(1),\n        withLatestFrom(viewport$)\n      )\n        .subscribe(([{ active }, { offset }]) => {\n          const tab = active.innerText.trim()\n          if (active.hasAttribute(\"data-md-switching\")) {\n            active.removeAttribute(\"data-md-switching\")\n\n          /* Determine viewport offset of active tab */\n          } else {\n            const y = el.offsetTop - offset.y\n\n            /* Passively activate other tabs */\n            for (const set of getElements(\"[data-tabs]\"))\n              for (const input of getElements<HTMLInputElement>(\n                \":scope > input\", set\n              )) {\n                const label = getElement(`label[for=\"${input.id}\"]`)\n                if (\n                  label !== active &&\n                  label.innerText.trim() === tab\n                ) {\n                  label.setAttribute(\"data-md-switching\", \"\")\n                  input.click()\n                  break\n                }\n              }\n\n            /* Bring active tab into view */\n            window.scrollTo({\n              top: el.offsetTop - y\n            })\n\n            /* Persist active tabs in local storage */\n            const tabs = __md_get<string[]>(\"__tabs\") || []\n            __md_set(\"__tabs\", [...new Set([tab, ...tabs])])\n          }\n        })\n\n    /* Pause media (audio, video) on switch - see https://bit.ly/3Bk6cel */\n    push$.pipe(takeUntil(done$))\n      .subscribe(() => {\n        for (const media of getElements<HTMLAudioElement>(\"audio, video\", el))\n          media.pause()\n      })\n\n    /* Create and return component */\n    return watchContentTabs(inputs)\n      .pipe(\n        tap(state => push$.next(state)),\n        finalize(() => push$.complete()),\n        map(state => ({ ref: el, ...state }))\n      )\n  })\n    .pipe(\n      subscribeOn(asyncScheduler)\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Observable, merge } from \"rxjs\"\n\nimport { feature } from \"~/_\"\nimport { Viewport, getElements } from \"~/browser\"\n\nimport { Component } from \"../../_\"\nimport {\n  Tooltip,\n  mountTooltip\n} from \"../../tooltip\"\nimport {\n  Annotation,\n  mountAnnotationBlock\n} from \"../annotation\"\nimport {\n  CodeBlock,\n  mountCodeBlock\n} from \"../code\"\nimport {\n  Details,\n  mountDetails\n} from \"../details\"\nimport {\n  Mermaid,\n  mountMermaid\n} from \"../mermaid\"\nimport {\n  DataTable,\n  mountDataTable\n} from \"../table\"\nimport {\n  ContentTabs,\n  mountContentTabs\n} from \"../tabs\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Content\n */\nexport type Content =\n  | Annotation\n  | CodeBlock\n  | ContentTabs\n  | DataTable\n  | Details\n  | Mermaid\n  | Tooltip\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  target$: Observable<HTMLElement>     /* Location target observable */\n  print$: Observable<boolean>          /* Media print observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount content\n *\n * This function mounts all components that are found in the content of the\n * actual article, including code blocks, data tables and details.\n *\n * @param el - Content element\n * @param options - Options\n *\n * @returns Content component observable\n */\nexport function mountContent(\n  el: HTMLElement, { viewport$, target$, print$ }: MountOptions\n): Observable<Component<Content>> {\n  return merge(\n\n    /* Annotations */\n    ...getElements(\".annotate:not(.highlight)\", el)\n      .map(child => mountAnnotationBlock(child, { target$, print$ })),\n\n    /* Code blocks */\n    ...getElements(\"pre:not(.mermaid) > code\", el)\n      .map(child => mountCodeBlock(child, { target$, print$ })),\n\n    /* Mermaid diagrams */\n    ...getElements(\"pre.mermaid\", el)\n      .map(child => mountMermaid(child)),\n\n    /* Data tables */\n    ...getElements(\"table:not([class])\", el)\n      .map(child => mountDataTable(child)),\n\n    /* Details */\n    ...getElements(\"details\", el)\n      .map(child => mountDetails(child, { target$, print$ })),\n\n    /* Content tabs */\n    ...getElements(\"[data-tabs]\", el)\n      .map(child => mountContentTabs(child, { viewport$, target$ })),\n\n    /* Tooltips */\n    ...getElements(\"[title]\", el)\n      .filter(() => feature(\"content.tooltips\"))\n      .map(child => mountTooltip(child))\n  )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  defer,\n  delay,\n  finalize,\n  map,\n  merge,\n  of,\n  switchMap,\n  tap\n} from \"rxjs\"\n\nimport { getElement } from \"~/browser\"\n\nimport { Component } from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Dialog\n */\nexport interface Dialog {\n  message: string                      /* Dialog message */\n  active: boolean                      /* Dialog is active */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  alert$: Subject<string>              /* Alert subject */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  alert$: Subject<string>              /* Alert subject */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch dialog\n *\n * @param _el - Dialog element\n * @param options - Options\n *\n * @returns Dialog observable\n */\nexport function watchDialog(\n  _el: HTMLElement, { alert$ }: WatchOptions\n): Observable<Dialog> {\n  return alert$\n    .pipe(\n      switchMap(message => merge(\n        of(true),\n        of(false).pipe(delay(2000))\n      )\n        .pipe(\n          map(active => ({ message, active }))\n        )\n      )\n    )\n}\n\n/**\n * Mount dialog\n *\n * This function reveals the dialog in the right corner when a new alert is\n * emitted through the subject that is passed as part of the options.\n *\n * @param el - Dialog element\n * @param options - Options\n *\n * @returns Dialog component observable\n */\nexport function mountDialog(\n  el: HTMLElement, options: MountOptions\n): Observable<Component<Dialog>> {\n  const inner = getElement(\".md-typeset\", el)\n  return defer(() => {\n    const push$ = new Subject<Dialog>()\n    push$.subscribe(({ message, active }) => {\n      el.classList.toggle(\"md-dialog--active\", active)\n      inner.textContent = message\n    })\n\n    /* Create and return component */\n    return watchDialog(el, options)\n      .pipe(\n        tap(state => push$.next(state)),\n        finalize(() => push$.complete()),\n        map(state => ({ ref: el, ...state }))\n      )\n  })\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  bufferCount,\n  combineLatest,\n  combineLatestWith,\n  defer,\n  distinctUntilChanged,\n  distinctUntilKeyChanged,\n  endWith,\n  filter,\n  from,\n  ignoreElements,\n  map,\n  mergeMap,\n  mergeWith,\n  of,\n  shareReplay,\n  startWith,\n  switchMap,\n  takeUntil\n} from \"rxjs\"\n\nimport { feature } from \"~/_\"\nimport {\n  Viewport,\n  getElements,\n  watchElementSize,\n  watchToggle\n} from \"~/browser\"\n\nimport { Component } from \"../../_\"\nimport { Main } from \"../../main\"\nimport {\n  Tooltip,\n  mountTooltip\n} from \"../../tooltip\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Header\n */\nexport interface Header {\n  height: number                       /* Header visible height */\n  hidden: boolean                      /* Header is hidden */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n  main$: Observable<Main>              /* Main area observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Compute whether the header is hidden\n *\n * If the user scrolls past a certain threshold, the header can be hidden when\n * scrolling down, and shown when scrolling up.\n *\n * @param options - Options\n *\n * @returns Toggle observable\n */\nfunction isHidden({ viewport$ }: WatchOptions): Observable<boolean> {\n  if (!feature(\"header.autohide\"))\n    return of(false)\n\n  /* Compute direction and turning point */\n  const direction$ = viewport$\n    .pipe(\n      map(({ offset: { y } }) => y),\n      bufferCount(2, 1),\n      map(([a, b]) => [a < b, b] as const),\n      distinctUntilKeyChanged(0)\n    )\n\n  /* Compute whether header should be hidden */\n  const hidden$ = combineLatest([viewport$, direction$])\n    .pipe(\n      filter(([{ offset }, [, y]]) => Math.abs(y - offset.y) > 100),\n      map(([, [direction]]) => direction),\n      distinctUntilChanged()\n    )\n\n  /* Compute threshold for hiding */\n  const search$ = watchToggle(\"search\")\n  return combineLatest([viewport$, search$])\n    .pipe(\n      map(([{ offset }, search]) => offset.y > 400 && !search),\n      distinctUntilChanged(),\n      switchMap(active => active ? hidden$ : of(false)),\n      startWith(false)\n    )\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch header\n *\n * @param el - Header element\n * @param options - Options\n *\n * @returns Header observable\n */\nexport function watchHeader(\n  el: HTMLElement, options: WatchOptions\n): Observable<Header> {\n  return defer(() => combineLatest([\n    watchElementSize(el),\n    isHidden(options)\n  ]))\n    .pipe(\n      map(([{ height }, hidden]) => ({\n        height,\n        hidden\n      })),\n      distinctUntilChanged((a, b) => (\n        a.height === b.height &&\n        a.hidden === b.hidden\n      )),\n      shareReplay(1)\n    )\n}\n\n/**\n * Mount header\n *\n * This function manages the different states of the header, i.e. whether it's\n * hidden or rendered with a shadow. This depends heavily on the main area.\n *\n * @param el - Header element\n * @param options - Options\n *\n * @returns Header component observable\n */\nexport function mountHeader(\n  el: HTMLElement, { header$, main$ }: MountOptions\n): Observable<Component<Header | Tooltip>> {\n  return defer(() => {\n    const push$ = new Subject<Main>()\n    const done$ = push$.pipe(ignoreElements(), endWith(true))\n    push$\n      .pipe(\n        distinctUntilKeyChanged(\"active\"),\n        combineLatestWith(header$)\n      )\n        .subscribe(([{ active }, { hidden }]) => {\n          el.classList.toggle(\"md-header--shadow\", active && !hidden)\n          el.hidden = hidden\n        })\n\n    /* Mount tooltips, if enabled */\n    const tooltips = from(getElements(\"[title]\", el))\n      .pipe(\n        filter(() => feature(\"content.tooltips\")),\n        mergeMap(child => mountTooltip(child))\n      )\n\n    /* Link to main area */\n    main$.subscribe(push$)\n\n    /* Create and return component */\n    return header$\n      .pipe(\n        takeUntil(done$),\n        map(state => ({ ref: el, ...state })),\n        mergeWith(tooltips.pipe(takeUntil(done$)))\n      )\n  })\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  EMPTY,\n  Observable,\n  Subject,\n  defer,\n  distinctUntilKeyChanged,\n  finalize,\n  map,\n  tap\n} from \"rxjs\"\n\nimport {\n  Viewport,\n  getElementSize,\n  getOptionalElement,\n  watchViewportAt\n} from \"~/browser\"\n\nimport { Component } from \"../../_\"\nimport { Header } from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Header\n */\nexport interface HeaderTitle {\n  active: boolean                      /* Header title is active */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch header title\n *\n * @param el - Heading element\n * @param options - Options\n *\n * @returns Header title observable\n */\nexport function watchHeaderTitle(\n  el: HTMLElement, { viewport$, header$ }: WatchOptions\n): Observable<HeaderTitle> {\n  return watchViewportAt(el, { viewport$, header$ })\n    .pipe(\n      map(({ offset: { y } }) => {\n        const { height } = getElementSize(el)\n        return {\n          active: y >= height\n        }\n      }),\n      distinctUntilKeyChanged(\"active\")\n    )\n}\n\n/**\n * Mount header title\n *\n * This function swaps the header title from the site title to the title of the\n * current page when the user scrolls past the first headline.\n *\n * @param el - Header title element\n * @param options - Options\n *\n * @returns Header title component observable\n */\nexport function mountHeaderTitle(\n  el: HTMLElement, options: MountOptions\n): Observable<Component<HeaderTitle>> {\n  return defer(() => {\n    const push$ = new Subject<HeaderTitle>()\n    push$.subscribe({\n\n      /* Handle emission */\n      next({ active }) {\n        el.classList.toggle(\"md-header__title--active\", active)\n      },\n\n      /* Handle complete */\n      complete() {\n        el.classList.remove(\"md-header__title--active\")\n      }\n    })\n\n    /* Obtain headline, if any */\n    const heading = getOptionalElement(\".md-content h1\")\n    if (typeof heading === \"undefined\")\n      return EMPTY\n\n    /* Create and return component */\n    return watchHeaderTitle(heading, options)\n      .pipe(\n        tap(state => push$.next(state)),\n        finalize(() => push$.complete()),\n        map(state => ({ ref: el, ...state }))\n      )\n  })\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  combineLatest,\n  distinctUntilChanged,\n  distinctUntilKeyChanged,\n  map,\n  switchMap\n} from \"rxjs\"\n\nimport {\n  Viewport,\n  watchElementSize\n} from \"~/browser\"\n\nimport { Header } from \"../header\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Main area\n */\nexport interface Main {\n  offset: number                       /* Main area top offset */\n  height: number                       /* Main area visible height */\n  active: boolean                      /* Main area is active */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch main area\n *\n * This function returns an observable that computes the visual parameters of\n * the main area which depends on the viewport vertical offset and height, as\n * well as the height of the header element, if the header is fixed.\n *\n * @param el - Main area element\n * @param options - Options\n *\n * @returns Main area observable\n */\nexport function watchMain(\n  el: HTMLElement, { viewport$, header$ }: WatchOptions\n): Observable<Main> {\n\n  /* Compute necessary adjustment for header */\n  const adjust$ = header$\n    .pipe(\n      map(({ height }) => height),\n      distinctUntilChanged()\n    )\n\n  /* Compute the main area's top and bottom borders */\n  const border$ = adjust$\n    .pipe(\n      switchMap(() => watchElementSize(el)\n        .pipe(\n          map(({ height }) => ({\n            top:    el.offsetTop,\n            bottom: el.offsetTop + height\n          })),\n          distinctUntilKeyChanged(\"bottom\")\n        )\n      )\n    )\n\n  /* Compute the main area's offset, visible height and if we scrolled past */\n  return combineLatest([adjust$, border$, viewport$])\n    .pipe(\n      map(([header, { top, bottom }, { offset: { y }, size: { height } }]) => {\n        height = Math.max(0, height\n          - Math.max(0, top    - y,  header)\n          - Math.max(0, height + y - bottom)\n        )\n        return {\n          offset: top - header,\n          height,\n          active: top - header <= y\n        }\n      }),\n      distinctUntilChanged((a, b) => (\n        a.offset === b.offset &&\n        a.height === b.height &&\n        a.active === b.active\n      ))\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  asyncScheduler,\n  defer,\n  finalize,\n  fromEvent,\n  map,\n  mergeMap,\n  observeOn,\n  of,\n  repeat,\n  shareReplay,\n  skip,\n  startWith,\n  takeUntil,\n  tap\n} from \"rxjs\"\n\nimport { getElements, watchMedia } from \"~/browser\"\nimport { h } from \"~/utilities\"\n\nimport {\n  Component,\n  getComponentElement\n} from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Palette colors\n */\nexport interface PaletteColor {\n  media?: string                       /* Media query */\n  scheme?: string                      /* Color scheme */\n  primary?: string                     /* Primary color */\n  accent?: string                      /* Accent color */\n}\n\n/**\n * Palette\n */\nexport interface Palette {\n  index: number                        /* Palette index */\n  color: PaletteColor                  /* Palette colors */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch color palette\n *\n * @param inputs - Color palette element\n *\n * @returns Color palette observable\n */\nexport function watchPalette(\n  inputs: HTMLInputElement[]\n): Observable<Palette> {\n  const current = __md_get<Palette>(\"__palette\") || {\n    index: inputs.findIndex(input => matchMedia(\n      input.getAttribute(\"data-md-color-media\")!\n    ).matches)\n  }\n\n  /* Emit changes in color palette */\n  const index = Math.max(0, Math.min(current.index, inputs.length - 1))\n  return of(...inputs)\n    .pipe(\n      mergeMap(input => fromEvent(input, \"change\").pipe(map(() => input))),\n      startWith(inputs[index]),\n      map(input => ({\n        index: inputs.indexOf(input),\n        color: {\n          media:   input.getAttribute(\"data-md-color-media\"),\n          scheme:  input.getAttribute(\"data-md-color-scheme\"),\n          primary: input.getAttribute(\"data-md-color-primary\"),\n          accent:  input.getAttribute(\"data-md-color-accent\")\n        }\n      } as Palette)),\n      shareReplay(1)\n    )\n}\n\n/**\n * Mount color palette\n *\n * @param el - Color palette element\n *\n * @returns Color palette component observable\n */\nexport function mountPalette(\n  el: HTMLElement\n): Observable<Component<Palette>> {\n  const inputs = getElements<HTMLInputElement>(\"input\", el)\n  const meta = h(\"meta\", { name: \"theme-color\" })\n  document.head.appendChild(meta)\n\n  // Add color scheme meta tag\n  const scheme = h(\"meta\", { name: \"color-scheme\" })\n  document.head.appendChild(scheme)\n\n  /* Mount component on subscription */\n  const media$ = watchMedia(\"(prefers-color-scheme: light)\")\n  return defer(() => {\n    const push$ = new Subject<Palette>()\n    push$.subscribe(palette => {\n      document.body.setAttribute(\"data-md-color-switching\", \"\")\n\n      /* Retrieve color palette for system preference */\n      if (palette.color.media === \"(prefers-color-scheme)\") {\n        const media = matchMedia(\"(prefers-color-scheme: light)\")\n        const input = document.querySelector(media.matches\n          ? \"[data-md-color-media='(prefers-color-scheme: light)']\"\n          : \"[data-md-color-media='(prefers-color-scheme: dark)']\"\n        )!\n\n        /* Retrieve colors for system preference */\n        palette.color.scheme  = input.getAttribute(\"data-md-color-scheme\")!\n        palette.color.primary = input.getAttribute(\"data-md-color-primary\")!\n        palette.color.accent  = input.getAttribute(\"data-md-color-accent\")!\n      }\n\n      /* Set color palette */\n      for (const [key, value] of Object.entries(palette.color))\n        document.body.setAttribute(`data-md-color-${key}`, value)\n\n      /* Set toggle visibility */\n      for (let index = 0; index < inputs.length; index++) {\n        const label = inputs[index].nextElementSibling\n        if (label instanceof HTMLElement)\n          label.hidden = palette.index !== index\n      }\n\n      /* Persist preference in local storage */\n      __md_set(\"__palette\", palette)\n    })\n\n    /* Update theme-color meta tag */\n    push$\n      .pipe(\n        map(() => {\n          const header = getComponentElement(\"header\")\n          const style  = window.getComputedStyle(header)\n\n          // Set color scheme\n          scheme.content = style.colorScheme\n\n          /* Return color in hexadecimal format */\n          return style.backgroundColor.match(/\\d+/g)!\n            .map(value => (+value).toString(16).padStart(2, \"0\"))\n            .join(\"\")\n        })\n      )\n        .subscribe(color => meta.content = `#${color}`)\n\n    /* Revert transition durations after color switch */\n    push$.pipe(observeOn(asyncScheduler))\n      .subscribe(() => {\n        document.body.removeAttribute(\"data-md-color-switching\")\n      })\n\n    /* Create and return component */\n    return watchPalette(inputs)\n      .pipe(\n        takeUntil(media$.pipe(skip(1))),\n        repeat(),\n        tap(state => push$.next(state)),\n        finalize(() => push$.complete()),\n        map(state => ({ ref: el, ...state }))\n      )\n  })\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  defer,\n  finalize,\n  map,\n  tap\n} from \"rxjs\"\n\nimport { Component } from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Progress indicator\n */\nexport interface Progress {\n  value: number                        // Progress value\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  progress$: Subject<number>           // Progress subject\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount progress indicator\n *\n * @param el - Progress indicator element\n * @param options - Options\n *\n * @returns Progress indicator component observable\n */\nexport function mountProgress(\n  el: HTMLElement, { progress$ }: MountOptions\n): Observable<Component<Progress>> {\n\n  // Mount component on subscription\n  return defer(() => {\n    const push$ = new Subject<Progress>()\n    push$.subscribe(({ value }) => {\n      el.style.setProperty(\"--md-progress-value\", `${value}`)\n    })\n\n    // Create and return component\n    return progress$\n      .pipe(\n        tap(value => push$.next({ value })),\n        finalize(() => push$.complete()),\n        map(value => ({ ref: el, value }))\n      )\n  })\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport ClipboardJS from \"clipboard\"\nimport {\n  Observable,\n  Subject,\n  map,\n  tap\n} from \"rxjs\"\n\nimport { translation } from \"~/_\"\nimport { getElement } from \"~/browser\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Setup options\n */\ninterface SetupOptions {\n  alert$: Subject<string>              /* Alert subject */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Extract text to copy\n *\n * @param el - HTML element\n *\n * @returns Extracted text\n */\nfunction extract(el: HTMLElement): string {\n  el.setAttribute(\"data-md-copying\", \"\")\n  const copy = el.closest(\"[data-copy]\")\n  const text = copy\n    ? copy.getAttribute(\"data-copy\")!\n    : el.innerText\n  el.removeAttribute(\"data-md-copying\")\n  return text.trimEnd()\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set up Clipboard.js integration\n *\n * @param options - Options\n */\nexport function setupClipboardJS(\n  { alert$ }: SetupOptions\n): void {\n  if (ClipboardJS.isSupported()) {\n    new Observable<ClipboardJS.Event>(subscriber => {\n      new ClipboardJS(\"[data-clipboard-target], [data-clipboard-text]\", {\n        text: el => (\n          el.getAttribute(\"data-clipboard-text\")! ||\n          extract(getElement(\n            el.getAttribute(\"data-clipboard-target\")!\n          ))\n        )\n      })\n        .on(\"success\", ev => subscriber.next(ev))\n    })\n      .pipe(\n        tap(ev => {\n          const trigger = ev.trigger as HTMLElement\n          trigger.focus()\n        }),\n        map(() => translation(\"clipboard.copied\"))\n      )\n        .subscribe(alert$)\n  }\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  catchError,\n  map,\n  of\n} from \"rxjs\"\n\nimport {\n  getElement,\n  getElements,\n  requestXML\n} from \"~/browser\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Sitemap, i.e. a list of URLs\n */\nexport type Sitemap = Map<string, URL[]>\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Resolve URL to the given base URL\n *\n * When serving the site with instant navigation, MkDocs will set the hostname\n * to the value as specified in `dev_addr`, but the browser allows for several\n * hostnames to be used: `localhost`, `127.0.0.1` or even `0.0.0.0`, depending\n * on configuration. This function resolves the URL to the given hostname.\n *\n * @param url - URL\n * @param base - Base URL\n *\n * @returns Resolved URL\n */\nfunction resolve(url: URL, base: URL) {\n  url.protocol = base.protocol\n  url.hostname = base.hostname\n  return url\n}\n\n/**\n * Extract sitemap from document\n *\n * This function extracts the URLs and alternate links from the document, and\n * associates alternate links to the original URL as found in `loc`, allowing\n * the browser to navigate to the correct page when switching languages. The\n * format of the sitemap is expected to adhere to:\n *\n * ``` xml\n * <urlset>\n *   <url>\n *     <loc>...</loc>\n *     <xhtml:link rel=\"alternate\" hreflang=\"en\" href=\"...\"/>\n *     <xhtml:link rel=\"alternate\" hreflang=\"de\" href=\"...\"/>\n *     ...\n *   </url>\n *   ...\n * </urlset>\n * ```\n *\n * @param document - Document\n * @param base - Base URL\n *\n * @returns Sitemap\n */\nfunction extract(document: Document, base: URL): Sitemap {\n  const sitemap: Sitemap = new Map()\n  for (const el of getElements(\"url\", document)) {\n    const url = getElement(\"loc\", el)\n\n    // Create entry for location and add it to the list of links\n    const links = [resolve(new URL(url.textContent!), base)]\n    sitemap.set(`${links[0]}`, links)\n\n    // Attach alternate links to current entry\n    for (const link of getElements(\"[rel=alternate]\", el)) {\n      const href = link.getAttribute(\"href\")\n      if (href != null)\n        links.push(resolve(new URL(href), base))\n    }\n  }\n\n  // Return sitemap\n  return sitemap\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Fetch the sitemap for the given base URL\n *\n * If a network or parsing error occurs, we just default to an empty sitemap,\n * which means the caller should fall back to regular navigation.\n *\n * @param base - Base URL\n *\n * @returns Sitemap observable\n */\nexport function fetchSitemap(base: URL | string): Observable<Sitemap> {\n  return requestXML(new URL(\"sitemap.xml\", base))\n    .pipe(\n      map(document => extract(document, new URL(base))),\n      catchError(() => of(new Map())),\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  EMPTY,\n  Observable,\n  Subject,\n  catchError,\n  combineLatestWith,\n  concat,\n  debounceTime,\n  distinctUntilChanged,\n  distinctUntilKeyChanged,\n  endWith,\n  fromEvent,\n  ignoreElements,\n  map,\n  merge,\n  of,\n  share,\n  switchMap,\n  tap,\n  withLatestFrom\n} from \"rxjs\"\n\nimport { configuration, feature } from \"~/_\"\nimport {\n  Viewport,\n  getElements,\n  getLocation,\n  getOptionalElement,\n  requestHTML,\n  setLocation,\n  setLocationHash\n} from \"~/browser\"\nimport { getComponentElement } from \"~/components\"\n\nimport { Sitemap, fetchSitemap } from \"../sitemap\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Setup options\n */\ninterface SetupOptions {\n  location$: Subject<URL>              // Location subject\n  viewport$: Observable<Viewport>      // Viewport observable\n  progress$: Subject<number>           // Progress subject\n}\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Handle clicks on internal URLs while skipping external URLs\n *\n * @param ev - Mouse event\n * @param sitemap - Sitemap\n *\n * @returns URL observable\n */\nfunction handle(\n  ev: MouseEvent, sitemap: Sitemap\n): Observable<URL> {\n  if (!(ev.target instanceof Element))\n    return EMPTY\n\n  // Skip, as target is not within a link - clicks on non-link elements are\n  // also captured, which we need to exclude from processing\n  const el = ev.target.closest(\"a\")\n  if (el === null)\n    return EMPTY\n\n  // Skip, as link opens in new window - we now know we have captured a click\n  // on a link, but the link either has a `target` property defined, or the\n  // user pressed the `meta` or `ctrl` key to open it in a new window. Thus,\n  // we need to filter this event as well.\n  if (el.target || ev.metaKey || ev.ctrlKey)\n    return EMPTY\n\n  // Next, we must check if the URL is relevant for us, i.e., if it's an\n  // internal link to a page that is managed by MkDocs. Only then we can be\n  // sure that the structure of the page to be loaded adheres to the current\n  // document structure and can subsequently be injected into it without doing\n  // a full reload. For this reason, we must canonicalize the URL by removing\n  // all search parameters and hash fragments.\n  const url = new URL(el.href)\n  url.search = url.hash = \"\"\n\n  // Skip, if URL is not included in the sitemap - this could be the case when\n  // linking between versions or languages, or to another page that the author\n  // included as part of the build, but that is not managed by MkDocs. In that\n  // case we must not continue with instant navigation.\n  if (!sitemap.has(`${url}`))\n    return EMPTY\n\n  // We now know that we have a link to an internal page, so we prevent the\n  // browser from navigation and emit the URL for instant navigation. Note that\n  // this also includes anchor links, which means we need to implement anchor\n  // positioning ourselves. The reason for this is that if we wouldn't manage\n  // anchor links as well, scroll restoration will not work correctly (e.g.\n  // following an anchor link and scrolling).\n  ev.preventDefault()\n  return of(new URL(el.href))\n}\n\n/**\n * Create a map of head elements for lookup and replacement\n *\n * @param document - Document\n *\n * @returns Tag map\n */\nfunction head(document: Document): Map<string, HTMLElement> {\n  const tags = new Map<string, HTMLElement>()\n  for (const el of getElements(\":scope > *\", document.head))\n    tags.set(el.outerHTML, el)\n\n  // Return tag map\n  return tags\n}\n\n/**\n * Resolve relative URLs in the given document\n *\n * This function resolves relative `href` and `src` attributes, which can belong\n * to all sorts of tags, like meta tags, links, images, scripts and more.\n *\n * @param document - Document\n *\n * @returns Document observable\n */\nfunction resolve(document: Document): Observable<Document> {\n  for (const el of getElements(\"[href], [src]\", document))\n    for (const key of [\"href\", \"src\"]) {\n      const value = el.getAttribute(key)\n      if (value && !/^(?:[a-z]+:)?\\/\\//i.test(value)) {\n        // @ts-expect-error - trick: self-assign to resolve URL\n        el[key] = el[key]\n        break\n      }\n    }\n\n  // Return document observable\n  return of(document)\n}\n\n/**\n * Inject the contents of a document into the current one\n *\n * @param next - Next document\n *\n * @returns Document observable\n */\nfunction inject(next: Document): Observable<Document> {\n  for (const selector of [\n    \"[data-md-component=announce]\",\n    \"[data-md-component=container]\",\n    \"[data-md-component=header-topic]\",\n    \"[data-md-component=outdated]\",\n    \"[data-md-component=logo]\",\n    \"[data-md-component=skip]\",\n    ...feature(\"navigation.tabs.sticky\")\n      ? [\"[data-md-component=tabs]\"]\n      : []\n  ]) {\n    const source = getOptionalElement(selector)\n    const target = getOptionalElement(selector, next)\n    if (\n      typeof source !== \"undefined\" &&\n      typeof target !== \"undefined\"\n    ) {\n      source.replaceWith(target)\n    }\n  }\n\n  // Update meta tags\n  const tags = head(document)\n  for (const [html, el] of head(next))\n    if (tags.has(html))\n      tags.delete(html)\n    else\n      document.head.appendChild(el)\n\n  // Remove meta tags that are not present in the new document\n  for (const el of tags.values()) {\n    const name = el.getAttribute(\"name\")\n    // @todo - find a better way to handle attributes we add dynamically in\n    // other components without mounting components on every navigation, as\n    // this might impact overall performance - see https://t.ly/ehp_O\n    if (name !== \"theme-color\" && name !== \"color-scheme\")\n      el.remove()\n  }\n\n  // After components and meta tags were replaced, re-evaluate scripts\n  // that were provided by the author as part of Markdown files\n  const container = getComponentElement(\"container\")\n  return concat(getElements(\"script\", container))\n    .pipe(\n      switchMap(el => {\n        const script = next.createElement(\"script\")\n        if (el.src) {\n          for (const name of el.getAttributeNames())\n            script.setAttribute(name, el.getAttribute(name)!)\n          el.replaceWith(script)\n\n          // Complete when script is loaded\n          return new Observable(observer => {\n            script.onload = () => observer.complete()\n          })\n\n        // Complete immediately\n        } else {\n          script.textContent = el.textContent\n          el.replaceWith(script)\n          return EMPTY\n        }\n      }),\n      ignoreElements(),\n      endWith(document)\n    )\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set up instant navigation\n *\n * This is a heavily orchestrated operation - see inline comments to learn how\n * this works with Material for MkDocs, and how you can hook into it.\n *\n * @param options - Options\n *\n * @returns Document observable\n */\nexport function setupInstantNavigation(\n  { location$, viewport$, progress$ }: SetupOptions\n): Observable<Document> {\n  const config = configuration()\n  if (location.protocol === \"file:\")\n    return EMPTY\n\n  // Load sitemap immediately, so we have it available when the user initiates\n  // the first navigation request without any perceivable delay\n  const sitemap$ = fetchSitemap(config.base)\n\n  // Since we might be on a slow connection, the user might trigger multiple\n  // instant navigation events that overlap. MkDocs produces relative URLs for\n  // all internal links, which becomes a problem in this case, because we need\n  // to change the base URL the moment the user clicks a link that should be\n  // intercepted in order to be consistent with popstate, which means that the\n  // base URL would now be incorrect when resolving another relative link from\n  // the same site. For this reason we always resolve all relative links to\n  // absolute links, so we can be sure this never happens.\n  of(document)\n    .subscribe(resolve)\n\n  // --------------------------------------------------------------------------\n  // Navigation interception\n  // --------------------------------------------------------------------------\n\n  // Intercept navigation - to keep the number of event listeners down we use\n  // the fact that uncaptured events bubble up to the body. This has the nice\n  // property that we don't need to detach and then re-attach event listeners\n  // when the document is replaced after a navigation event.\n  const instant$ =\n    fromEvent<MouseEvent>(document.body, \"click\")\n      .pipe(\n        combineLatestWith(sitemap$),\n        switchMap(([ev, sitemap]) => handle(ev, sitemap)),\n        share()\n      )\n\n  // Intercept history change events, e.g. when the user uses the browser's\n  // back or forward buttons, and emit new location for fetching and parsing\n  const history$ =\n    fromEvent<PopStateEvent>(window, \"popstate\")\n      .pipe(\n        map(getLocation),\n        share()\n      )\n\n  // While it would be better UX to defer navigation events until the document\n  // is fully fetched and parsed, we must schedule it here to synchronize with\n  // popstate events, as they are emitted immediately. Moreover we need to\n  // store the current viewport offset for scroll restoration later on.\n  instant$.pipe(withLatestFrom(viewport$))\n    .subscribe(([url, { offset }]) => {\n      history.replaceState(offset, \"\")\n      history.pushState(null, \"\", url)\n    })\n\n  // Emit URLs that should be fetched via instant navigation on location subject\n  // which was passed into this function. The state of instant navigation can be\n  // intercepted by other parts of the application, which can synchronously back\n  // up or restore state before or after instant navigation happens.\n  merge(instant$, history$)\n    .subscribe(location$)\n\n  // --------------------------------------------------------------------------\n  // Fetching and parsing\n  // --------------------------------------------------------------------------\n\n  // Fetch document - we deduplicate requests to the same location, so we don't\n  // end up with multiple requests for the same page. We use `switchMap`, since\n  // we want to cancel the previous request when a new one is triggered, which\n  // is automatically handled by the observable returned by `request`. This is\n  // essential to ensure a good user experience, as we don't want to load pages\n  // that are not needed anymore, e.g., when the user clicks multiple links in\n  // quick succession or on slow connections. If the request fails for some\n  // reason, we fall back and use regular navigation, forcing a reload.\n  const document$ =\n    location$.pipe(\n      distinctUntilKeyChanged(\"pathname\"),\n      switchMap(url => requestHTML(url, { progress$ })\n        .pipe(\n          catchError(() => {\n            setLocation(url, true)\n            return EMPTY\n          })\n        )\n      ),\n\n      // The document was successfully fetched and parsed, so we can inject its\n      // contents into the currently active document\n      switchMap(resolve),\n      switchMap(inject),\n      share()\n    )\n\n  // --------------------------------------------------------------------------\n  // Scroll restoration\n  // --------------------------------------------------------------------------\n\n  // Handle scroll restoration - we must restore the viewport offset after the\n  // document has been fetched and injected, and every time the user clicks an\n  // anchor that leads to an element on the same page, which might also happen\n  // when the user uses the back or forward button.\n  merge(\n    document$.pipe(withLatestFrom(location$, (_, url) => url)),\n\n    // Handle instant navigation events that are triggered by the user clicking\n    // on an anchor link with a hash fragment different from the current one, as\n    // well as from popstate events, which are emitted when the user navigates\n    // back and forth between pages. We use a two-layered subscription to scope\n    // the scroll restoration to the current page, as we don't need to restore\n    // the viewport offset when the user navigates to a different page, as this\n    // is already handled by the previous observable.\n    location$.pipe(\n      distinctUntilKeyChanged(\"pathname\"),\n      switchMap(() => location$),\n      distinctUntilKeyChanged(\"hash\"),\n    ),\n\n    // Handle instant navigation events that are triggered by the user clicking\n    // on an anchor link with the same hash fragment as the current one in the\n    // URL. Is is essential that we only intercept those from instant navigation\n    // events and not from history change events, or we'll end up in and endless\n    // loop. The top-level history entry must be removed, as it will be replaced\n    // with a new one, which would otherwise lead to a duplicate entry.\n    location$.pipe(\n      distinctUntilChanged((a, b) => (\n        a.pathname === b.pathname &&\n        a.hash     === b.hash\n      )),\n      switchMap(() => instant$),\n      tap(() => history.back())\n    )\n  )\n    .subscribe(url => {\n\n      // Check if the current history entry has a state, which happens when the\n      // user presses the back or forward button to visit a page we've already\n      // seen. If there's no state, it means a new page was visited and we must\n      // scroll to the top, unless an anchor is given.\n      if (history.state !== null || !url.hash) {\n        window.scrollTo(0, history.state?.y ?? 0)\n      } else {\n        history.scrollRestoration = \"auto\"\n        setLocationHash(url.hash)\n        history.scrollRestoration = \"manual\"\n      }\n    })\n\n  // Disable scroll restoration when an instant navigation event occurs, so the\n  // browser does not immediately set the viewport offset to the prior history\n  // entry, scrolling to the position on the same page, which would look odd.\n  // Instead, we manually restore the position once the page has loaded.\n  location$.subscribe(() => {\n    history.scrollRestoration = \"manual\"\n  })\n\n  // Enable scroll restoration before window unloads - this is essential to\n  // ensure that full reloads (F5) restore the viewport offset correctly. If\n  // only popstate events wouldn't reset the viewport offset prior to their\n  // emission, we could just reset this in popstate. Meh.\n  fromEvent(window, \"beforeunload\")\n    .subscribe(() => {\n      history.scrollRestoration = \"auto\"\n    })\n\n  // Track viewport offset, so we can restore it when the user navigates back\n  // and forth between pages. Note that this must be debounced and cannot be\n  // done in popstate, as popstate has already removed the entry from the\n  // history, which means it is too late.\n  viewport$.pipe(\n    distinctUntilKeyChanged(\"offset\"),\n    debounceTime(100)\n  )\n    .subscribe(({ offset }) => {\n      history.replaceState(offset, \"\")\n    })\n\n  // Return document observable\n  return document$\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport escapeHTML from \"escape-html\"\n\nimport { SearchConfig } from \"../config\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search highlight function\n *\n * @param value - Value\n *\n * @returns Highlighted value\n */\nexport type SearchHighlightFn = (value: string) => string\n\n/**\n * Search highlight factory function\n *\n * @param query - Query value\n *\n * @returns Search highlight function\n */\nexport type SearchHighlightFactoryFn = (query: string) => SearchHighlightFn\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Create a search highlighter\n *\n * @param config - Search configuration\n *\n * @returns Search highlight factory function\n */\nexport function setupSearchHighlighter(\n  config: SearchConfig\n): SearchHighlightFactoryFn {\n  // Hack: temporarily remove pure lookaheads and lookbehinds\n  const regex = config.separator.split(\"|\").map(term => {\n    const temp = term.replace(/(\\(\\?[!=<][^)]+\\))/g, \"\")\n    return temp.length === 0 ? \"\uFFFD\" : term\n  })\n    .join(\"|\")\n\n  const separator = new RegExp(regex, \"img\")\n  const highlight = (_: unknown, data: string, term: string) => {\n    return `${data}<mark data-md-highlight>${term}</mark>`\n  }\n\n  /* Return factory function */\n  return (query: string) => {\n    query = query\n      .replace(/[\\s*+\\-:~^]+/g, \" \")\n      .trim()\n\n    /* Create search term match expression */\n    const match = new RegExp(`(^|${config.separator}|)(${\n      query\n        .replace(/[|\\\\{}()[\\]^$+*?.-]/g, \"\\\\$&\")\n        .replace(separator, \"|\")\n    })`, \"img\")\n\n    /* Highlight string value */\n    return value => escapeHTML(value)\n      .replace(match, highlight)\n      .replace(/<\\/mark>(\\s+)<mark[^>]*>/img, \"$1\")\n  }\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A RTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { SearchResult } from \"../../_\"\nimport { SearchIndex } from \"../../config\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search message type\n */\nexport const enum SearchMessageType {\n  SETUP,                               /* Search index setup */\n  READY,                               /* Search index ready */\n  QUERY,                               /* Search query */\n  RESULT                               /* Search results */\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Message containing the data necessary to setup the search index\n */\nexport interface SearchSetupMessage {\n  type: SearchMessageType.SETUP        /* Message type */\n  data: SearchIndex                    /* Message data */\n}\n\n/**\n * Message indicating the search index is ready\n */\nexport interface SearchReadyMessage {\n  type: SearchMessageType.READY        /* Message type */\n}\n\n/**\n * Message containing a search query\n */\nexport interface SearchQueryMessage {\n  type: SearchMessageType.QUERY        /* Message type */\n  data: string                         /* Message data */\n}\n\n/**\n * Message containing results for a search query\n */\nexport interface SearchResultMessage {\n  type: SearchMessageType.RESULT       /* Message type */\n  data: SearchResult                   /* Message data */\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Message exchanged with the search worker\n */\nexport type SearchMessage =\n  | SearchSetupMessage\n  | SearchReadyMessage\n  | SearchQueryMessage\n  | SearchResultMessage\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Type guard for search ready messages\n *\n * @param message - Search worker message\n *\n * @returns Test result\n */\nexport function isSearchReadyMessage(\n  message: SearchMessage\n): message is SearchReadyMessage {\n  return message.type === SearchMessageType.READY\n}\n\n/**\n * Type guard for search result messages\n *\n * @param message - Search worker message\n *\n * @returns Test result\n */\nexport function isSearchResultMessage(\n  message: SearchMessage\n): message is SearchResultMessage {\n  return message.type === SearchMessageType.RESULT\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A RTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  ObservableInput,\n  Subject,\n  first,\n  merge,\n  of,\n  switchMap\n} from \"rxjs\"\n\nimport { feature } from \"~/_\"\nimport { watchToggle, watchWorker } from \"~/browser\"\n\nimport { SearchIndex } from \"../../config\"\nimport {\n  SearchMessage,\n  SearchMessageType\n} from \"../message\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set up search worker\n *\n * This function creates and initializes a web worker that is used for search,\n * so that the user interface doesn't freeze. In general, the application does\n * not care how search is implemented, as long as the web worker conforms to\n * the format expected by the application as defined in `SearchMessage`. This\n * allows the author to implement custom search functionality, by providing a\n * custom web worker via configuration.\n *\n * Material for MkDocs' built-in search implementation makes use of Lunr.js, an\n * efficient and fast implementation for client-side search. Leveraging a tiny\n * iframe-based web worker shim, search is even supported for the `file://`\n * protocol, enabling search for local non-hosted builds.\n *\n * If the protocol is `file://`, search initialization is deferred to mitigate\n * freezing, as it's now synchronous by design - see https://bit.ly/3C521EO\n *\n * @see https://bit.ly/3igvtQv - How to implement custom search\n *\n * @param url - Worker URL\n * @param index$ - Search index observable input\n *\n * @returns Search worker\n */\nexport function setupSearchWorker(\n  url: string, index$: ObservableInput<SearchIndex>\n): Subject<SearchMessage> {\n  const worker$ = watchWorker<SearchMessage>(url)\n  merge(\n    of(location.protocol !== \"file:\"),\n    watchToggle(\"search\")\n  )\n    .pipe(\n      first(active => active),\n      switchMap(() => index$)\n    )\n      .subscribe(({ config, docs }) => worker$.next({\n        type: SearchMessageType.SETUP,\n        data: {\n          config,\n          docs,\n          options: {\n            suggest: feature(\"search.suggest\")\n          }\n        }\n      }))\n\n  /* Return search worker */\n  return worker$\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  EMPTY,\n  Subject,\n  catchError,\n  combineLatest,\n  filter,\n  fromEvent,\n  map,\n  of,\n  switchMap,\n  withLatestFrom\n} from \"rxjs\"\n\nimport { configuration } from \"~/_\"\nimport {\n  getElement,\n  getLocation,\n  requestJSON,\n  setLocation\n} from \"~/browser\"\nimport { getComponentElements } from \"~/components\"\nimport {\n  Version,\n  renderVersionSelector\n} from \"~/templates\"\n\nimport { fetchSitemap } from \"../sitemap\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Setup options\n */\ninterface SetupOptions {\n  document$: Subject<Document>         /* Document subject */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set up version selector\n *\n * @param options - Options\n */\nexport function setupVersionSelector(\n  { document$ }: SetupOptions\n): void {\n  const config = configuration()\n  const versions$ = requestJSON<Version[]>(\n    new URL(\"../versions.json\", config.base)\n  )\n    .pipe(\n      catchError(() => EMPTY) // @todo refactor instant loading\n    )\n\n  /* Determine current version */\n  const current$ = versions$\n    .pipe(\n      map(versions => {\n        const [, current] = config.base.match(/([^/]+)\\/?$/)!\n        return versions.find(({ version, aliases }) => (\n          version === current || aliases.includes(current)\n        )) || versions[0]\n      })\n    )\n\n  /* Intercept inter-version navigation */\n  versions$\n    .pipe(\n      map(versions => new Map(versions.map(version => [\n        `${new URL(`../${version.version}/`, config.base)}`,\n        version\n      ]))),\n      switchMap(urls => fromEvent<MouseEvent>(document.body, \"click\")\n        .pipe(\n          filter(ev => !ev.metaKey && !ev.ctrlKey),\n          withLatestFrom(current$),\n          switchMap(([ev, current]) => {\n            if (ev.target instanceof Element) {\n              const el = ev.target.closest(\"a\")\n              if (el && !el.target && urls.has(el.href)) {\n                const url = el.href\n                // This is a temporary hack to detect if a version inside the\n                // version selector or on another part of the site was clicked.\n                // If we're inside the version selector, we definitely want to\n                // find the same page, as we might have different deployments\n                // due to aliases. However, if we're outside the version\n                // selector, we must abort here, because we might otherwise\n                // interfere with instant navigation. We need to refactor this\n                // at some point together with instant navigation.\n                //\n                // See https://github.com/squidfunk/mkdocs-material/issues/4012\n                if (!ev.target.closest(\".md-version\")) {\n                  const version = urls.get(url)!\n                  if (version === current)\n                    return EMPTY\n                }\n                ev.preventDefault()\n                return of(url)\n              }\n            }\n            return EMPTY\n          }),\n          switchMap(url => {\n            const { version } = urls.get(url)!\n            return fetchSitemap(new URL(url))\n              .pipe(\n                map(sitemap => {\n                  const location = getLocation()\n                  const path = location.href.replace(config.base, \"\")\n                  return sitemap.has(path.split(\"#\")[0])\n                    ? new URL(`../${version}/${path}`, config.base)\n                    : new URL(url)\n                })\n              )\n          })\n        )\n      )\n    )\n      .subscribe(url => setLocation(url, true))\n\n  /* Render version selector and warning */\n  combineLatest([versions$, current$])\n    .subscribe(([versions, current]) => {\n      const topic = getElement(\".md-header__topic\")\n      topic.appendChild(renderVersionSelector(versions, current))\n    })\n\n  /* Integrate outdated version banner with instant navigation */\n  document$.pipe(switchMap(() => current$))\n    .subscribe(current => {\n\n      /* Check if version state was already determined */\n      let outdated = __md_get(\"__outdated\", sessionStorage)\n      if (outdated === null) {\n        outdated = true\n\n        /* Obtain and normalize default versions */\n        let ignored = config.version?.default || \"latest\"\n        if (!Array.isArray(ignored))\n          ignored = [ignored]\n\n        /* Check if version is considered a default */\n        main: for (const ignore of ignored)\n          for (const version of current.aliases.concat(current.version))\n            if (new RegExp(ignore, \"i\").test(version)) {\n              outdated = false\n              break main\n            }\n\n        /* Persist version state in session storage */\n        __md_set(\"__outdated\", outdated, sessionStorage)\n      }\n\n      /* Unhide outdated version banner */\n      if (outdated)\n        for (const warning of getComponentElements(\"outdated\"))\n          warning.hidden = false\n    })\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  combineLatest,\n  distinctUntilChanged,\n  distinctUntilKeyChanged,\n  endWith,\n  finalize,\n  first,\n  fromEvent,\n  ignoreElements,\n  map,\n  merge,\n  shareReplay,\n  takeUntil,\n  tap\n} from \"rxjs\"\n\nimport {\n  getElement,\n  getLocation,\n  setToggle,\n  watchElementFocus,\n  watchToggle\n} from \"~/browser\"\nimport {\n  SearchMessage,\n  SearchMessageType,\n  isSearchReadyMessage\n} from \"~/integrations\"\n\nimport { Component } from \"../../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search query\n */\nexport interface SearchQuery {\n  value: string                        /* Query value */\n  focus: boolean                       /* Query focus */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  worker$: Subject<SearchMessage>      /* Search worker */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  worker$: Subject<SearchMessage>      /* Search worker */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch search query\n *\n * Note that the focus event which triggers re-reading the current query value\n * is delayed by `1ms` so the input's empty state is allowed to propagate.\n *\n * @param el - Search query element\n * @param options - Options\n *\n * @returns Search query observable\n */\nexport function watchSearchQuery(\n  el: HTMLInputElement, { worker$ }: WatchOptions\n): Observable<SearchQuery> {\n\n  /* Support search deep linking */\n  const { searchParams } = getLocation()\n  if (searchParams.has(\"q\")) {\n    setToggle(\"search\", true)\n\n    /* Set query from parameter */\n    el.value = searchParams.get(\"q\")!\n    el.focus()\n\n    /* Remove query parameter on close */\n    watchToggle(\"search\")\n      .pipe(\n        first(active => !active)\n      )\n        .subscribe(() => {\n          const url = getLocation()\n          url.searchParams.delete(\"q\")\n          history.replaceState({}, \"\", `${url}`)\n        })\n  }\n\n  /* Intercept focus and input events */\n  const focus$ = watchElementFocus(el)\n  const value$ = merge(\n    worker$.pipe(first(isSearchReadyMessage)),\n    fromEvent(el, \"keyup\"),\n    focus$\n  )\n    .pipe(\n      map(() => el.value),\n      distinctUntilChanged()\n    )\n\n  /* Combine into single observable */\n  return combineLatest([value$, focus$])\n    .pipe(\n      map(([value, focus]) => ({ value, focus })),\n      shareReplay(1)\n    )\n}\n\n/**\n * Mount search query\n *\n * @param el - Search query element\n * @param options - Options\n *\n * @returns Search query component observable\n */\nexport function mountSearchQuery(\n  el: HTMLInputElement, { worker$ }: MountOptions\n): Observable<Component<SearchQuery, HTMLInputElement>> {\n  const push$ = new Subject<SearchQuery>()\n  const done$ = push$.pipe(ignoreElements(), endWith(true))\n\n  /* Handle value change */\n  combineLatest([\n    worker$.pipe(first(isSearchReadyMessage)),\n    push$\n  ], (_, query) => query)\n    .pipe(\n      distinctUntilKeyChanged(\"value\")\n    )\n      .subscribe(({ value }) => worker$.next({\n        type: SearchMessageType.QUERY,\n        data: value\n      }))\n\n  /* Handle focus change */\n  push$\n    .pipe(\n      distinctUntilKeyChanged(\"focus\")\n    )\n      .subscribe(({ focus }) => {\n        if (focus)\n          setToggle(\"search\", focus)\n      })\n\n  /* Handle reset */\n  fromEvent(el.form!, \"reset\")\n    .pipe(\n      takeUntil(done$)\n    )\n      .subscribe(() => el.focus())\n\n  // Focus search query on label click - note that this is necessary to bring\n  // up the keyboard on iOS and other mobile platforms, as the search dialog is\n  // not visible at first, and programatically focusing an input element must\n  // be triggered by a user interaction - see https://t.ly/Cb30n\n  const label = getElement(\"header [for=__search]\")\n  fromEvent(label, \"click\")\n    .subscribe(() => el.focus())\n\n  /* Create and return component */\n  return watchSearchQuery(el, { worker$ })\n    .pipe(\n      tap(state => push$.next(state)),\n      finalize(() => push$.complete()),\n      map(state => ({ ref: el, ...state })),\n      shareReplay(1)\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  EMPTY,\n  Observable,\n  Subject,\n  bufferCount,\n  filter,\n  finalize,\n  first,\n  fromEvent,\n  map,\n  merge,\n  mergeMap,\n  of,\n  share,\n  skipUntil,\n  switchMap,\n  takeUntil,\n  tap,\n  withLatestFrom,\n  zipWith\n} from \"rxjs\"\n\nimport { translation } from \"~/_\"\nimport {\n  getElement,\n  getOptionalElement,\n  watchElementBoundary,\n  watchToggle\n} from \"~/browser\"\nimport {\n  SearchMessage,\n  SearchResult,\n  isSearchReadyMessage,\n  isSearchResultMessage\n} from \"~/integrations\"\nimport { renderSearchResultItem } from \"~/templates\"\nimport { round } from \"~/utilities\"\n\nimport { Component } from \"../../_\"\nimport { SearchQuery } from \"../query\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  query$: Observable<SearchQuery>      /* Search query observable */\n  worker$: Subject<SearchMessage>      /* Search worker */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount search result list\n *\n * This function performs a lazy rendering of the search results, depending on\n * the vertical offset of the search result container.\n *\n * @param el - Search result list element\n * @param options - Options\n *\n * @returns Search result list component observable\n */\nexport function mountSearchResult(\n  el: HTMLElement, { worker$, query$ }: MountOptions\n): Observable<Component<SearchResult>> {\n  const push$ = new Subject<SearchResult>()\n  const boundary$ = watchElementBoundary(el.parentElement!)\n    .pipe(\n      filter(Boolean)\n    )\n\n  /* Retrieve container */\n  const container = el.parentElement!\n\n  /* Retrieve nested components */\n  const meta = getElement(\":scope > :first-child\", el)\n  const list = getElement(\":scope > :last-child\", el)\n\n  /* Reveal to accessibility tree \u2013 see https://bit.ly/3iAA7t8 */\n  watchToggle(\"search\")\n    .subscribe(active => list.setAttribute(\n      \"role\", active ? \"list\" : \"presentation\"\n    ))\n\n  /* Update search result metadata */\n  push$\n    .pipe(\n      withLatestFrom(query$),\n      skipUntil(worker$.pipe(first(isSearchReadyMessage)))\n    )\n      .subscribe(([{ items }, { value }]) => {\n        switch (items.length) {\n\n          /* No results */\n          case 0:\n            meta.textContent = value.length\n              ? translation(\"search.result.none\")\n              : translation(\"search.result.placeholder\")\n            break\n\n          /* One result */\n          case 1:\n            meta.textContent = translation(\"search.result.one\")\n            break\n\n          /* Multiple result */\n          default:\n            const count = round(items.length)\n            meta.textContent = translation(\"search.result.other\", count)\n        }\n      })\n\n  /* Render search result item */\n  const render$ = push$\n    .pipe(\n      tap(() => list.innerHTML = \"\"),\n      switchMap(({ items }) => merge(\n        of(...items.slice(0, 10)),\n        of(...items.slice(10))\n          .pipe(\n            bufferCount(4),\n            zipWith(boundary$),\n            switchMap(([chunk]) => chunk)\n          )\n      )),\n      map(renderSearchResultItem),\n      share()\n    )\n\n  /* Update search result list */\n  render$.subscribe(item => list.appendChild(item))\n  render$\n    .pipe(\n      mergeMap(item => {\n        const details = getOptionalElement(\"details\", item)\n        if (typeof details === \"undefined\")\n          return EMPTY\n\n        /* Keep position of details element stable */\n        return fromEvent(details, \"toggle\")\n          .pipe(\n            takeUntil(push$),\n            map(() => details)\n          )\n      })\n    )\n      .subscribe(details => {\n        if (\n          details.open === false &&\n          details.offsetTop <= container.scrollTop\n        )\n          container.scrollTo({ top: details.offsetTop })\n      })\n\n  /* Filter search result message */\n  const result$ = worker$\n    .pipe(\n      filter(isSearchResultMessage),\n      map(({ data }) => data)\n    )\n\n  /* Create and return component */\n  return result$\n    .pipe(\n      tap(state => push$.next(state)),\n      finalize(() => push$.complete()),\n      map(state => ({ ref: el, ...state }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  endWith,\n  finalize,\n  fromEvent,\n  ignoreElements,\n  map,\n  takeUntil,\n  tap\n} from \"rxjs\"\n\nimport { getLocation } from \"~/browser\"\n\nimport { Component } from \"../../_\"\nimport { SearchQuery } from \"../query\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search sharing\n */\nexport interface SearchShare {\n  url: URL                             /* Deep link for sharing */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  query$: Observable<SearchQuery>      /* Search query observable */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  query$: Observable<SearchQuery>      /* Search query observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount search sharing\n *\n * @param _el - Search sharing element\n * @param options - Options\n *\n * @returns Search sharing observable\n */\nexport function watchSearchShare(\n  _el: HTMLElement, { query$ }: WatchOptions\n): Observable<SearchShare> {\n  return query$\n    .pipe(\n      map(({ value }) => {\n        const url = getLocation()\n        url.hash = \"\"\n\n        /* Compute readable query strings */\n        value = value\n          .replace(/\\s+/g, \"+\")        /* Collapse whitespace */\n          .replace(/&/g, \"%26\")        /* Escape '&' character */\n          .replace(/=/g, \"%3D\")        /* Escape '=' character */\n\n        /* Replace query string */\n        url.search = `q=${value}`\n        return { url }\n      })\n    )\n}\n\n/**\n * Mount search sharing\n *\n * @param el - Search sharing element\n * @param options - Options\n *\n * @returns Search sharing component observable\n */\nexport function mountSearchShare(\n  el: HTMLAnchorElement, options: MountOptions\n): Observable<Component<SearchShare>> {\n  const push$ = new Subject<SearchShare>()\n  const done$ = push$.pipe(ignoreElements(), endWith(true))\n  push$.subscribe(({ url }) => {\n    el.setAttribute(\"data-clipboard-text\", el.href)\n    el.href = `${url}`\n  })\n\n  /* Prevent following of link */\n  fromEvent(el, \"click\")\n    .pipe(\n      takeUntil(done$)\n    )\n      .subscribe(ev => ev.preventDefault())\n\n  /* Create and return component */\n  return watchSearchShare(el, options)\n    .pipe(\n      tap(state => push$.next(state)),\n      finalize(() => push$.complete()),\n      map(state => ({ ref: el, ...state }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  asyncScheduler,\n  combineLatestWith,\n  distinctUntilChanged,\n  filter,\n  finalize,\n  fromEvent,\n  map,\n  merge,\n  observeOn,\n  tap\n} from \"rxjs\"\n\nimport { Keyboard } from \"~/browser\"\nimport {\n  SearchMessage,\n  SearchResult,\n  isSearchResultMessage\n} from \"~/integrations\"\n\nimport { Component, getComponentElement } from \"../../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search suggestions\n */\nexport interface SearchSuggest {}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  keyboard$: Observable<Keyboard>      /* Keyboard observable */\n  worker$: Subject<SearchMessage>      /* Search worker */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount search suggestions\n *\n * This function will perform a lazy rendering of the search results, depending\n * on the vertical offset of the search result container.\n *\n * @param el - Search result list element\n * @param options - Options\n *\n * @returns Search result list component observable\n */\nexport function mountSearchSuggest(\n  el: HTMLElement, { worker$, keyboard$ }: MountOptions\n): Observable<Component<SearchSuggest>> {\n  const push$ = new Subject<SearchResult>()\n\n  /* Retrieve query component and track all changes */\n  const query  = getComponentElement(\"search-query\")\n  const query$ = merge(\n    fromEvent(query, \"keydown\"),\n    fromEvent(query, \"focus\")\n  )\n    .pipe(\n      observeOn(asyncScheduler),\n      map(() => query.value),\n      distinctUntilChanged(),\n    )\n\n  /* Update search suggestions */\n  push$\n    .pipe(\n      combineLatestWith(query$),\n      map(([{ suggest }, value]) => {\n        const words = value.split(/([\\s-]+)/)\n        if (suggest?.length && words[words.length - 1]) {\n          const last = suggest[suggest.length - 1]\n          if (last.startsWith(words[words.length - 1]))\n            words[words.length - 1] = last\n        } else {\n          words.length = 0\n        }\n        return words\n      })\n    )\n      .subscribe(words => el.innerHTML = words\n        .join(\"\")\n        .replace(/\\s/g, \"&nbsp;\")\n      )\n\n  /* Set up search keyboard handlers */\n  keyboard$\n    .pipe(\n      filter(({ mode }) => mode === \"search\")\n    )\n      .subscribe(key => {\n        switch (key.type) {\n\n          /* Right arrow: accept current suggestion */\n          case \"ArrowRight\":\n            if (\n              el.innerText.length &&\n              query.selectionStart === query.value.length\n            )\n              query.value = el.innerText\n            break\n        }\n      })\n\n  /* Filter search result message */\n  const result$ = worker$\n    .pipe(\n      filter(isSearchResultMessage),\n      map(({ data }) => data)\n    )\n\n  /* Create and return component */\n  return result$\n    .pipe(\n      tap(state => push$.next(state)),\n      finalize(() => push$.complete()),\n      map(() => ({ ref: el }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  NEVER,\n  Observable,\n  ObservableInput,\n  filter,\n  fromEvent,\n  merge,\n  mergeWith\n} from \"rxjs\"\n\nimport { configuration } from \"~/_\"\nimport {\n  Keyboard,\n  getActiveElement,\n  getElements,\n  setToggle\n} from \"~/browser\"\nimport {\n  SearchIndex,\n  SearchResult,\n  setupSearchWorker\n} from \"~/integrations\"\n\nimport {\n  Component,\n  getComponentElement,\n  getComponentElements\n} from \"../../_\"\nimport {\n  SearchQuery,\n  mountSearchQuery\n} from \"../query\"\nimport { mountSearchResult } from \"../result\"\nimport {\n  SearchShare,\n  mountSearchShare\n} from \"../share\"\nimport {\n  SearchSuggest,\n  mountSearchSuggest\n} from \"../suggest\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search\n */\nexport type Search =\n  | SearchQuery\n  | SearchResult\n  | SearchShare\n  | SearchSuggest\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  index$: ObservableInput<SearchIndex> /* Search index observable */\n  keyboard$: Observable<Keyboard>      /* Keyboard observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount search\n *\n * This function sets up the search functionality, including the underlying\n * web worker and all keyboard bindings.\n *\n * @param el - Search element\n * @param options - Options\n *\n * @returns Search component observable\n */\nexport function mountSearch(\n  el: HTMLElement, { index$, keyboard$ }: MountOptions\n): Observable<Component<Search>> {\n  const config = configuration()\n  try {\n    const worker$ = setupSearchWorker(config.search, index$)\n\n    /* Retrieve query and result components */\n    const query  = getComponentElement(\"search-query\", el)\n    const result = getComponentElement(\"search-result\", el)\n\n    /* Always close search on result selection */\n    fromEvent<PointerEvent>(el, \"click\")\n      .pipe(\n        filter(({ target }) => (\n          target instanceof Element && !!target.closest(\"a\")\n        ))\n      )\n        .subscribe(() => setToggle(\"search\", false))\n\n    /* Set up search keyboard handlers */\n    keyboard$\n      .pipe(\n        filter(({ mode }) => mode === \"search\")\n      )\n        .subscribe(key => {\n          const active = getActiveElement()\n          switch (key.type) {\n\n            /* Enter: go to first (best) result */\n            case \"Enter\":\n              if (active === query) {\n                const anchors = new Map<HTMLAnchorElement, number>()\n                for (const anchor of getElements<HTMLAnchorElement>(\n                  \":first-child [href]\", result\n                )) {\n                  const article = anchor.firstElementChild!\n                  anchors.set(anchor, parseFloat(\n                    article.getAttribute(\"data-md-score\")!\n                  ))\n                }\n\n                /* Go to result with highest score, if any */\n                if (anchors.size) {\n                  const [[best]] = [...anchors].sort(([, a], [, b]) => b - a)\n                  best.click()\n                }\n\n                /* Otherwise omit form submission */\n                key.claim()\n              }\n              break\n\n            /* Escape or Tab: close search */\n            case \"Escape\":\n            case \"Tab\":\n              setToggle(\"search\", false)\n              query.blur()\n              break\n\n            /* Vertical arrows: select previous or next search result */\n            case \"ArrowUp\":\n            case \"ArrowDown\":\n              if (typeof active === \"undefined\") {\n                query.focus()\n              } else {\n                const els = [query, ...getElements(\n                  \":not(details) > [href], summary, details[open] [href]\",\n                  result\n                )]\n                const i = Math.max(0, (\n                  Math.max(0, els.indexOf(active)) + els.length + (\n                    key.type === \"ArrowUp\" ? -1 : +1\n                  )\n                ) % els.length)\n                els[i].focus()\n              }\n\n              /* Prevent scrolling of page */\n              key.claim()\n              break\n\n            /* All other keys: hand to search query */\n            default:\n              if (query !== getActiveElement())\n                query.focus()\n          }\n        })\n\n    /* Set up global keyboard handlers */\n    keyboard$\n      .pipe(\n        filter(({ mode }) => mode === \"global\")\n      )\n        .subscribe(key => {\n          switch (key.type) {\n\n            /* Open search and select query */\n            case \"f\":\n            case \"s\":\n            case \"/\":\n              query.focus()\n              query.select()\n\n              /* Prevent scrolling of page */\n              key.claim()\n              break\n          }\n        })\n\n    /* Create and return component */\n    const query$ = mountSearchQuery(query, { worker$ })\n    return merge(\n      query$,\n      mountSearchResult(result, { worker$, query$ })\n    )\n      .pipe(\n        mergeWith(\n\n          /* Search sharing */\n          ...getComponentElements(\"search-share\", el)\n            .map(child => mountSearchShare(child, { query$ })),\n\n          /* Search suggestions */\n          ...getComponentElements(\"search-suggest\", el)\n            .map(child => mountSearchSuggest(child, { worker$, keyboard$ }))\n        )\n      )\n\n  /* Gracefully handle broken search */\n  } catch (err) {\n    el.hidden = true\n    return NEVER\n  }\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  ObservableInput,\n  combineLatest,\n  filter,\n  map,\n  startWith\n} from \"rxjs\"\n\nimport { getLocation } from \"~/browser\"\nimport {\n  SearchIndex,\n  setupSearchHighlighter\n} from \"~/integrations\"\nimport { h } from \"~/utilities\"\n\nimport { Component } from \"../../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search highlighting\n */\nexport interface SearchHighlight {\n  nodes: Map<ChildNode, string>        /* Map of replacements */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  index$: ObservableInput<SearchIndex> /* Search index observable */\n  location$: Observable<URL>           /* Location observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount search highlighting\n *\n * @param el - Content element\n * @param options - Options\n *\n * @returns Search highlighting component observable\n */\nexport function mountSearchHiglight(\n  el: HTMLElement, { index$, location$ }: MountOptions\n): Observable<Component<SearchHighlight>> {\n  return combineLatest([\n    index$,\n    location$\n      .pipe(\n        startWith(getLocation()),\n        filter(url => !!url.searchParams.get(\"h\"))\n      )\n  ])\n    .pipe(\n      map(([index, url]) => setupSearchHighlighter(index.config)(\n        url.searchParams.get(\"h\")!\n      )),\n      map(fn => {\n        const nodes = new Map<ChildNode, string>()\n\n        /* Traverse text nodes and collect matches */\n        const it = document.createNodeIterator(el, NodeFilter.SHOW_TEXT)\n        for (let node = it.nextNode(); node; node = it.nextNode()) {\n          if (node.parentElement?.offsetHeight) {\n            const original = node.textContent!\n            const replaced = fn(original)\n            if (replaced.length > original.length)\n              nodes.set(node as ChildNode, replaced)\n          }\n        }\n\n        /* Replace original nodes with matches */\n        for (const [node, text] of nodes) {\n          const { childNodes } = h(\"span\", null, text)\n          node.replaceWith(...Array.from(childNodes))\n        }\n\n        /* Return component */\n        return { ref: el, nodes }\n      })\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  animationFrameScheduler,\n  asyncScheduler,\n  auditTime,\n  combineLatest,\n  defer,\n  distinctUntilChanged,\n  endWith,\n  finalize,\n  first,\n  from,\n  fromEvent,\n  ignoreElements,\n  map,\n  mergeMap,\n  observeOn,\n  takeUntil,\n  tap,\n  withLatestFrom\n} from \"rxjs\"\n\nimport {\n  Viewport,\n  getElement,\n  getElementOffset,\n  getElementSize,\n  getElements\n} from \"~/browser\"\n\nimport { Component } from \"../_\"\nimport { Header } from \"../header\"\nimport { Main } from \"../main\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Sidebar\n */\nexport interface Sidebar {\n  height: number                       /* Sidebar height */\n  locked: boolean                      /* Sidebar is locked */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  main$: Observable<Main>              /* Main area observable */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n  main$: Observable<Main>              /* Main area observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch sidebar\n *\n * This function returns an observable that computes the visual parameters of\n * the sidebar which depends on the vertical viewport offset, as well as the\n * height of the main area. When the page is scrolled beyond the header, the\n * sidebar is locked and fills the remaining space.\n *\n * @param el - Sidebar element\n * @param options - Options\n *\n * @returns Sidebar observable\n */\nexport function watchSidebar(\n  el: HTMLElement, { viewport$, main$ }: WatchOptions\n): Observable<Sidebar> {\n  const parent = el.closest<HTMLElement>(\".md-grid\")!\n  const adjust =\n    parent.offsetTop -\n    parent.parentElement!.offsetTop\n\n  /* Compute the sidebar's available height and if it should be locked */\n  return combineLatest([main$, viewport$])\n    .pipe(\n      map(([{ offset, height }, { offset: { y } }]) => {\n        height = height\n          + Math.min(adjust, Math.max(0, y - offset))\n          - adjust\n        return {\n          height,\n          locked: y >= offset + adjust\n        }\n      }),\n      distinctUntilChanged((a, b) => (\n        a.height === b.height &&\n        a.locked === b.locked\n      ))\n    )\n}\n\n/**\n * Mount sidebar\n *\n * This function doesn't set the height of the actual sidebar, but of its first\n * child \u2013 the `.md-sidebar__scrollwrap` element in order to mitigiate jittery\n * sidebars when the footer is scrolled into view. At some point we switched\n * from `absolute` / `fixed` positioning to `sticky` positioning, significantly\n * reducing jitter in some browsers (respectively Firefox and Safari) when\n * scrolling from the top. However, top-aligned sticky positioning means that\n * the sidebar snaps to the bottom when the end of the container is reached.\n * This is what leads to the mentioned jitter, as the sidebar's height may be\n * updated too slowly.\n *\n * This behaviour can be mitigiated by setting the height of the sidebar to `0`\n * while preserving the padding, and the height on its first element.\n *\n * @param el - Sidebar element\n * @param options - Options\n *\n * @returns Sidebar component observable\n */\nexport function mountSidebar(\n  el: HTMLElement, { header$, ...options }: MountOptions\n): Observable<Component<Sidebar>> {\n  const inner = getElement(\".md-sidebar__scrollwrap\", el)\n  const { y } = getElementOffset(inner)\n  return defer(() => {\n    const push$ = new Subject<Sidebar>()\n    const done$ = push$.pipe(ignoreElements(), endWith(true))\n    const next$ = push$\n      .pipe(\n        auditTime(0, animationFrameScheduler)\n      )\n\n    /* Update sidebar height and offset */\n    next$.pipe(withLatestFrom(header$))\n      .subscribe({\n\n        /* Handle emission */\n        next([{ height }, { height: offset }]) {\n          inner.style.height = `${height - 2 * y}px`\n          el.style.top       = `${offset}px`\n        },\n\n        /* Handle complete */\n        complete() {\n          inner.style.height = \"\"\n          el.style.top       = \"\"\n        }\n      })\n\n    /* Bring active item into view on initial load */\n    next$.pipe(first())\n      .subscribe(() => {\n        for (const item of getElements(\".md-nav__link--active[href]\", el)) {\n          if (!item.clientHeight) // skip invisible toc in left sidebar\n            continue\n          const container = item.closest<HTMLElement>(\".md-sidebar__scrollwrap\")!\n          if (typeof container !== \"undefined\") {\n            const offset = item.offsetTop - container.offsetTop\n            const { height } = getElementSize(container)\n            container.scrollTo({\n              top: offset - height / 2\n            })\n          }\n        }\n      })\n\n    /* Handle accessibility for expandable items, see https://bit.ly/3jaod9p */\n    from(getElements<HTMLLabelElement>(\"label[tabindex]\", el))\n      .pipe(\n        mergeMap(label => fromEvent(label, \"click\")\n          .pipe(\n            observeOn(asyncScheduler),\n            map(() => label),\n            takeUntil(done$)\n          )\n        )\n      )\n        .subscribe(label => {\n          const input = getElement<HTMLInputElement>(`[id=\"${label.htmlFor}\"]`)\n          const nav = getElement(`[aria-labelledby=\"${label.id}\"]`)\n          nav.setAttribute(\"aria-expanded\", `${input.checked}`)\n        })\n\n    /* Create and return component */\n    return watchSidebar(el, options)\n      .pipe(\n        tap(state => push$.next(state)),\n        finalize(() => push$.complete()),\n        map(state => ({ ref: el, ...state }))\n      )\n  })\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Repo, User } from \"github-types\"\nimport {\n  EMPTY,\n  Observable,\n  catchError,\n  defaultIfEmpty,\n  map,\n  zip\n} from \"rxjs\"\n\nimport { requestJSON } from \"~/browser\"\n\nimport { SourceFacts } from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * GitHub release (partial)\n */\ninterface Release {\n  tag_name: string                     /* Tag name */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Fetch GitHub repository facts\n *\n * @param user - GitHub user or organization\n * @param repo - GitHub repository\n *\n * @returns Repository facts observable\n */\nexport function fetchSourceFactsFromGitHub(\n  user: string, repo?: string\n): Observable<SourceFacts> {\n  if (typeof repo !== \"undefined\") {\n    const url = `https://api.github.com/repos/${user}/${repo}`\n    return zip(\n\n      /* Fetch version */\n      requestJSON<Release>(`${url}/releases/latest`)\n        .pipe(\n          catchError(() => EMPTY), // @todo refactor instant loading\n          map(release => ({\n            version: release.tag_name\n          })),\n          defaultIfEmpty({})\n        ),\n\n      /* Fetch stars and forks */\n      requestJSON<Repo>(url)\n        .pipe(\n          catchError(() => EMPTY), // @todo refactor instant loading\n          map(info => ({\n            stars: info.stargazers_count,\n            forks: info.forks_count\n          })),\n          defaultIfEmpty({})\n        )\n    )\n      .pipe(\n        map(([release, info]) => ({ ...release, ...info }))\n      )\n\n  /* User or organization */\n  } else {\n    const url = `https://api.github.com/users/${user}`\n    return requestJSON<User>(url)\n      .pipe(\n        map(info => ({\n          repositories: info.public_repos\n        })),\n        defaultIfEmpty({})\n      )\n  }\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { ProjectSchema } from \"gitlab\"\nimport {\n  EMPTY,\n  Observable,\n  catchError,\n  defaultIfEmpty,\n  map\n} from \"rxjs\"\n\nimport { requestJSON } from \"~/browser\"\n\nimport { SourceFacts } from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Fetch GitLab repository facts\n *\n * @param base - GitLab base\n * @param project - GitLab project\n *\n * @returns Repository facts observable\n */\nexport function fetchSourceFactsFromGitLab(\n  base: string, project: string\n): Observable<SourceFacts> {\n  const url = `https://${base}/api/v4/projects/${encodeURIComponent(project)}`\n  return requestJSON<ProjectSchema>(url)\n    .pipe(\n      catchError(() => EMPTY), // @todo refactor instant loading\n      map(({ star_count, forks_count }) => ({\n        stars: star_count,\n        forks: forks_count\n      })),\n      defaultIfEmpty({})\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { EMPTY, Observable } from \"rxjs\"\n\nimport { fetchSourceFactsFromGitHub } from \"../github\"\nimport { fetchSourceFactsFromGitLab } from \"../gitlab\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Repository facts for repositories\n */\nexport interface RepositoryFacts {\n  stars?: number                       /* Number of stars */\n  forks?: number                       /* Number of forks */\n  version?: string                     /* Latest version */\n}\n\n/**\n * Repository facts for organizations\n */\nexport interface OrganizationFacts {\n  repositories?: number                /* Number of repositories */\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Repository facts\n */\nexport type SourceFacts =\n  | RepositoryFacts\n  | OrganizationFacts\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Fetch repository facts\n *\n * @param url - Repository URL\n *\n * @returns Repository facts observable\n */\nexport function fetchSourceFacts(\n  url: string\n): Observable<SourceFacts> {\n\n  /* Try to match GitHub repository */\n  let match = url.match(/^.+github\\.com\\/([^/]+)\\/?([^/]+)?/i)\n  if (match) {\n    const [, user, repo] = match\n    return fetchSourceFactsFromGitHub(user, repo)\n  }\n\n  /* Try to match GitLab repository */\n  match = url.match(/^.+?([^/]*gitlab[^/]+)\\/(.+?)\\/?$/i)\n  if (match) {\n    const [, base, slug] = match\n    return fetchSourceFactsFromGitLab(base, slug)\n  }\n\n  /* Fallback */\n  return EMPTY\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  EMPTY,\n  Observable,\n  Subject,\n  catchError,\n  defer,\n  filter,\n  finalize,\n  map,\n  of,\n  shareReplay,\n  tap\n} from \"rxjs\"\n\nimport { getElement } from \"~/browser\"\nimport { ConsentDefaults } from \"~/components/consent\"\nimport { renderSourceFacts } from \"~/templates\"\n\nimport {\n  Component,\n  getComponentElements\n} from \"../../_\"\nimport {\n  SourceFacts,\n  fetchSourceFacts\n} from \"../facts\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Repository information\n */\nexport interface Source {\n  facts: SourceFacts                   /* Repository facts */\n}\n\n/* ----------------------------------------------------------------------------\n * Data\n * ------------------------------------------------------------------------- */\n\n/**\n * Repository information observable\n */\nlet fetch$: Observable<Source>\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch repository information\n *\n * This function tries to read the repository facts from session storage, and\n * if unsuccessful, fetches them from the underlying provider.\n *\n * @param el - Repository information element\n *\n * @returns Repository information observable\n */\nexport function watchSource(\n  el: HTMLAnchorElement\n): Observable<Source> {\n  return fetch$ ||= defer(() => {\n    const cached = __md_get<SourceFacts>(\"__source\", sessionStorage)\n    if (cached) {\n      return of(cached)\n    } else {\n\n      /* Check if consent is configured and was given */\n      const els = getComponentElements(\"consent\")\n      if (els.length) {\n        const consent = __md_get<ConsentDefaults>(\"__consent\")\n        if (!(consent && consent.github))\n          return EMPTY\n      }\n\n      /* Fetch repository facts */\n      return fetchSourceFacts(el.href)\n        .pipe(\n          tap(facts => __md_set(\"__source\", facts, sessionStorage))\n        )\n    }\n  })\n    .pipe(\n      catchError(() => EMPTY),\n      filter(facts => Object.keys(facts).length > 0),\n      map(facts => ({ facts })),\n      shareReplay(1)\n    )\n}\n\n/**\n * Mount repository information\n *\n * @param el - Repository information element\n *\n * @returns Repository information component observable\n */\nexport function mountSource(\n  el: HTMLAnchorElement\n): Observable<Component<Source>> {\n  const inner = getElement(\":scope > :last-child\", el)\n  return defer(() => {\n    const push$ = new Subject<Source>()\n    push$.subscribe(({ facts }) => {\n      inner.appendChild(renderSourceFacts(facts))\n      inner.classList.add(\"md-source__repository--active\")\n    })\n\n    /* Create and return component */\n    return watchSource(el)\n      .pipe(\n        tap(state => push$.next(state)),\n        finalize(() => push$.complete()),\n        map(state => ({ ref: el, ...state }))\n      )\n  })\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  defer,\n  distinctUntilKeyChanged,\n  finalize,\n  map,\n  of,\n  switchMap,\n  tap\n} from \"rxjs\"\n\nimport { feature } from \"~/_\"\nimport {\n  Viewport,\n  watchElementSize,\n  watchViewportAt\n} from \"~/browser\"\n\nimport { Component } from \"../_\"\nimport { Header } from \"../header\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Navigation tabs\n */\nexport interface Tabs {\n  hidden: boolean                      /* Navigation tabs are hidden */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch navigation tabs\n *\n * @param el - Navigation tabs element\n * @param options - Options\n *\n * @returns Navigation tabs observable\n */\nexport function watchTabs(\n  el: HTMLElement, { viewport$, header$ }: WatchOptions\n): Observable<Tabs> {\n  return watchElementSize(document.body)\n    .pipe(\n      switchMap(() => watchViewportAt(el, { header$, viewport$ })),\n      map(({ offset: { y } }) => {\n        return {\n          hidden: y >= 10\n        }\n      }),\n      distinctUntilKeyChanged(\"hidden\")\n    )\n}\n\n/**\n * Mount navigation tabs\n *\n * This function hides the navigation tabs when scrolling past the threshold\n * and makes them reappear in a nice CSS animation when scrolling back up.\n *\n * @param el - Navigation tabs element\n * @param options - Options\n *\n * @returns Navigation tabs component observable\n */\nexport function mountTabs(\n  el: HTMLElement, options: MountOptions\n): Observable<Component<Tabs>> {\n  return defer(() => {\n    const push$ = new Subject<Tabs>()\n    push$.subscribe({\n\n      /* Handle emission */\n      next({ hidden }) {\n        el.hidden = hidden\n      },\n\n      /* Handle complete */\n      complete() {\n        el.hidden = false\n      }\n    })\n\n    /* Create and return component */\n    return (\n      feature(\"navigation.tabs.sticky\")\n        ? of({ hidden: false })\n        : watchTabs(el, options)\n    )\n      .pipe(\n        tap(state => push$.next(state)),\n        finalize(() => push$.complete()),\n        map(state => ({ ref: el, ...state }))\n      )\n  })\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  asyncScheduler,\n  bufferCount,\n  combineLatestWith,\n  debounceTime,\n  defer,\n  distinctUntilChanged,\n  distinctUntilKeyChanged,\n  endWith,\n  filter,\n  finalize,\n  ignoreElements,\n  map,\n  merge,\n  observeOn,\n  of,\n  repeat,\n  scan,\n  share,\n  skip,\n  startWith,\n  switchMap,\n  takeUntil,\n  tap,\n  withLatestFrom\n} from \"rxjs\"\n\nimport { feature } from \"~/_\"\nimport {\n  Viewport,\n  getElement,\n  getElementContainer,\n  getElementSize,\n  getElements,\n  getLocation,\n  getOptionalElement,\n  watchElementSize\n} from \"~/browser\"\n\nimport {\n  Component,\n  getComponentElement\n} from \"../_\"\nimport { Header } from \"../header\"\nimport { Main } from \"../main\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Table of contents\n */\nexport interface TableOfContents {\n  prev: HTMLAnchorElement[][]          /* Anchors (previous) */\n  next: HTMLAnchorElement[][]          /* Anchors (next) */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n  main$: Observable<Main>              /* Main area observable */\n  target$: Observable<HTMLElement>     /* Location target observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch table of contents\n *\n * This is effectively a scroll spy implementation which will account for the\n * fixed header and automatically re-calculate anchor offsets when the viewport\n * is resized. The returned observable will only emit if the table of contents\n * needs to be repainted.\n *\n * This implementation tracks an anchor element's entire path starting from its\n * level up to the top-most anchor element, e.g. `[h3, h2, h1]`. Although the\n * Material theme currently doesn't make use of this information, it enables\n * the styling of the entire hierarchy through customization.\n *\n * Note that the current anchor is the last item of the `prev` anchor list.\n *\n * @param el - Table of contents element\n * @param options - Options\n *\n * @returns Table of contents observable\n */\nexport function watchTableOfContents(\n  el: HTMLElement, { viewport$, header$ }: WatchOptions\n): Observable<TableOfContents> {\n  const table = new Map<HTMLAnchorElement, HTMLElement>()\n\n  /* Compute anchor-to-target mapping */\n  const anchors = getElements<HTMLAnchorElement>(\".md-nav__link\", el)\n  for (const anchor of anchors) {\n    const id = decodeURIComponent(anchor.hash.substring(1))\n    const target = getOptionalElement(`[id=\"${id}\"]`)\n    if (typeof target !== \"undefined\")\n      table.set(anchor, target)\n  }\n\n  /* Compute necessary adjustment for header */\n  const adjust$ = header$\n    .pipe(\n      distinctUntilKeyChanged(\"height\"),\n      map(({ height }) => {\n        const main = getComponentElement(\"main\")\n        const grid = getElement(\":scope > :first-child\", main)\n        return height + 0.8 * (\n          grid.offsetTop -\n          main.offsetTop\n        )\n      }),\n      share()\n    )\n\n  /* Compute partition of previous and next anchors */\n  const partition$ = watchElementSize(document.body)\n    .pipe(\n      distinctUntilKeyChanged(\"height\"),\n\n      /* Build index to map anchor paths to vertical offsets */\n      switchMap(body => defer(() => {\n        let path: HTMLAnchorElement[] = []\n        return of([...table].reduce((index, [anchor, target]) => {\n          while (path.length) {\n            const last = table.get(path[path.length - 1])!\n            if (last.tagName >= target.tagName) {\n              path.pop()\n            } else {\n              break\n            }\n          }\n\n          /* If the current anchor is hidden, continue with its parent */\n          let offset = target.offsetTop\n          while (!offset && target.parentElement) {\n            target = target.parentElement\n            offset = target.offsetTop\n          }\n\n          /* Fix anchor offsets in tables - see https://bit.ly/3CUFOcn */\n          let parent = target.offsetParent as HTMLElement\n          for (; parent; parent = parent.offsetParent as HTMLElement)\n            offset += parent.offsetTop\n\n          /* Map reversed anchor path to vertical offset */\n          return index.set(\n            [...path = [...path, anchor]].reverse(),\n            offset\n          )\n        }, new Map<HTMLAnchorElement[], number>()))\n      })\n        .pipe(\n\n          /* Sort index by vertical offset (see https://bit.ly/30z6QSO) */\n          map(index => new Map([...index].sort(([, a], [, b]) => a - b))),\n          combineLatestWith(adjust$),\n\n          /* Re-compute partition when viewport offset changes */\n          switchMap(([index, adjust]) => viewport$\n            .pipe(\n              scan(([prev, next], { offset: { y }, size }) => {\n                const last = y + size.height >= Math.floor(body.height)\n\n                /* Look forward */\n                while (next.length) {\n                  const [, offset] = next[0]\n                  if (offset - adjust < y || last) {\n                    prev = [...prev, next.shift()!]\n                  } else {\n                    break\n                  }\n                }\n\n                /* Look backward */\n                while (prev.length) {\n                  const [, offset] = prev[prev.length - 1]\n                  if (offset - adjust >= y && !last) {\n                    next = [prev.pop()!, ...next]\n                  } else {\n                    break\n                  }\n                }\n\n                /* Return partition */\n                return [prev, next]\n              }, [[], [...index]]),\n              distinctUntilChanged((a, b) => (\n                a[0] === b[0] &&\n                a[1] === b[1]\n              ))\n            )\n          )\n        )\n      )\n    )\n\n  /* Compute and return anchor list migrations */\n  return partition$\n    .pipe(\n      map(([prev, next]) => ({\n        prev: prev.map(([path]) => path),\n        next: next.map(([path]) => path)\n      })),\n\n      /* Extract anchor list migrations */\n      startWith({ prev: [], next: [] }),\n      bufferCount(2, 1),\n      map(([a, b]) => {\n\n        /* Moving down */\n        if (a.prev.length < b.prev.length) {\n          return {\n            prev: b.prev.slice(Math.max(0, a.prev.length - 1), b.prev.length),\n            next: []\n          }\n\n        /* Moving up */\n        } else {\n          return {\n            prev: b.prev.slice(-1),\n            next: b.next.slice(0, b.next.length - a.next.length)\n          }\n        }\n      })\n    )\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Mount table of contents\n *\n * @param el - Table of contents element\n * @param options - Options\n *\n * @returns Table of contents component observable\n */\nexport function mountTableOfContents(\n  el: HTMLElement, { viewport$, header$, main$, target$ }: MountOptions\n): Observable<Component<TableOfContents>> {\n  return defer(() => {\n    const push$ = new Subject<TableOfContents>()\n    const done$ = push$.pipe(ignoreElements(), endWith(true))\n    push$.subscribe(({ prev, next }) => {\n\n      /* Look forward */\n      for (const [anchor] of next) {\n        anchor.classList.remove(\"md-nav__link--passed\")\n        anchor.classList.remove(\"md-nav__link--active\")\n      }\n\n      /* Look backward */\n      for (const [index, [anchor]] of prev.entries()) {\n        anchor.classList.add(\"md-nav__link--passed\")\n        anchor.classList.toggle(\n          \"md-nav__link--active\",\n          index === prev.length - 1\n        )\n      }\n    })\n\n    /* Set up following, if enabled */\n    if (feature(\"toc.follow\")) {\n\n      /* Toggle smooth scrolling only for anchor clicks */\n      const smooth$ = merge(\n        viewport$.pipe(debounceTime(1), map(() => undefined)),\n        viewport$.pipe(debounceTime(250), map(() => \"smooth\" as const))\n      )\n\n      /* Bring active anchor into view */ // @todo: refactor\n      push$\n        .pipe(\n          filter(({ prev }) => prev.length > 0),\n          combineLatestWith(main$.pipe(observeOn(asyncScheduler))),\n          withLatestFrom(smooth$)\n        )\n          .subscribe(([[{ prev }], behavior]) => {\n            const [anchor] = prev[prev.length - 1]\n            if (anchor.offsetHeight) {\n\n              /* Retrieve overflowing container and scroll */\n              const container = getElementContainer(anchor)\n              if (typeof container !== \"undefined\") {\n                const offset = anchor.offsetTop - container.offsetTop\n                const { height } = getElementSize(container)\n                container.scrollTo({\n                  top: offset - height / 2,\n                  behavior\n                })\n              }\n            }\n          })\n    }\n\n    /* Set up anchor tracking, if enabled */\n    if (feature(\"navigation.tracking\"))\n      viewport$\n        .pipe(\n          takeUntil(done$),\n          distinctUntilKeyChanged(\"offset\"),\n          debounceTime(250),\n          skip(1),\n          takeUntil(target$.pipe(skip(1))),\n          repeat({ delay: 250 }),\n          withLatestFrom(push$)\n        )\n          .subscribe(([, { prev }]) => {\n            const url = getLocation()\n\n            /* Set hash fragment to active anchor */\n            const anchor = prev[prev.length - 1]\n            if (anchor && anchor.length) {\n              const [active] = anchor\n              const { hash } = new URL(active.href)\n              if (url.hash !== hash) {\n                url.hash = hash\n                history.replaceState({}, \"\", `${url}`)\n              }\n\n            /* Reset anchor when at the top */\n            } else {\n              url.hash = \"\"\n              history.replaceState({}, \"\", `${url}`)\n            }\n          })\n\n    /* Create and return component */\n    return watchTableOfContents(el, { viewport$, header$ })\n      .pipe(\n        tap(state => push$.next(state)),\n        finalize(() => push$.complete()),\n        map(state => ({ ref: el, ...state }))\n      )\n  })\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  bufferCount,\n  combineLatest,\n  distinctUntilChanged,\n  distinctUntilKeyChanged,\n  endWith,\n  finalize,\n  fromEvent,\n  ignoreElements,\n  map,\n  repeat,\n  skip,\n  takeUntil,\n  tap\n} from \"rxjs\"\n\nimport { Viewport } from \"~/browser\"\n\nimport { Component } from \"../_\"\nimport { Header } from \"../header\"\nimport { Main } from \"../main\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Back-to-top button\n */\nexport interface BackToTop {\n  hidden: boolean                      /* Back-to-top button is hidden */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  main$: Observable<Main>              /* Main area observable */\n  target$: Observable<HTMLElement>     /* Location target observable */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n  main$: Observable<Main>              /* Main area observable */\n  target$: Observable<HTMLElement>     /* Location target observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch back-to-top\n *\n * @param _el - Back-to-top element\n * @param options - Options\n *\n * @returns Back-to-top observable\n */\nexport function watchBackToTop(\n  _el: HTMLElement, { viewport$, main$, target$ }: WatchOptions\n): Observable<BackToTop> {\n\n  /* Compute direction */\n  const direction$ = viewport$\n    .pipe(\n      map(({ offset: { y } }) => y),\n      bufferCount(2, 1),\n      map(([a, b]) => a > b && b > 0),\n      distinctUntilChanged()\n    )\n\n  /* Compute whether main area is active */\n  const active$ = main$\n    .pipe(\n      map(({ active }) => active)\n    )\n\n  /* Compute threshold for hiding */\n  return combineLatest([active$, direction$])\n    .pipe(\n      map(([active, direction]) => !(active && direction)),\n      distinctUntilChanged(),\n      takeUntil(target$.pipe(skip(1))),\n      endWith(true),\n      repeat({ delay: 250 }),\n      map(hidden => ({ hidden }))\n    )\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Mount back-to-top\n *\n * @param el - Back-to-top element\n * @param options - Options\n *\n * @returns Back-to-top component observable\n */\nexport function mountBackToTop(\n  el: HTMLElement, { viewport$, header$, main$, target$ }: MountOptions\n): Observable<Component<BackToTop>> {\n  const push$ = new Subject<BackToTop>()\n  const done$ = push$.pipe(ignoreElements(), endWith(true))\n  push$.subscribe({\n\n    /* Handle emission */\n    next({ hidden }) {\n      el.hidden = hidden\n      if (hidden) {\n        el.setAttribute(\"tabindex\", \"-1\")\n        el.blur()\n      } else {\n        el.removeAttribute(\"tabindex\")\n      }\n    },\n\n    /* Handle complete */\n    complete() {\n      el.style.top = \"\"\n      el.hidden = true\n      el.removeAttribute(\"tabindex\")\n    }\n  })\n\n  /* Watch header height */\n  header$\n    .pipe(\n      takeUntil(done$),\n      distinctUntilKeyChanged(\"height\")\n    )\n      .subscribe(({ height }) => {\n        el.style.top = `${height + 16}px`\n      })\n\n  /* Go back to top */\n  fromEvent(el, \"click\")\n    .subscribe(ev => {\n      ev.preventDefault()\n      window.scrollTo({ top: 0 })\n    })\n\n  /* Create and return component */\n  return watchBackToTop(el, { viewport$, main$, target$ })\n    .pipe(\n      tap(state => push$.next(state)),\n      finalize(() => push$.complete()),\n      map(state => ({ ref: el, ...state }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  filter,\n  finalize,\n  map,\n  mergeMap,\n  skip,\n  switchMap,\n  take,\n  takeUntil\n} from \"rxjs\"\n\nimport {\n  getElements,\n  watchElementVisibility\n} from \"~/browser\"\nimport { mountTooltip } from \"~/components\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Patch options\n */\ninterface PatchOptions {\n  document$: Observable<Document>      /* Document observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Patch ellipsis\n *\n * This function will fetch all elements that are shortened with ellipsis, and\n * filter those which are visible. Once they become visible, they stay in that\n * state, even though they may be hidden again. This optimization is necessary\n * to reduce pressure on the browser, with elements fading in and out of view.\n *\n * @param options - Options\n */\nexport function patchEllipsis(\n  { document$ }: PatchOptions\n): void {\n  document$\n    .pipe(\n      switchMap(() => getElements(\".md-ellipsis\")),\n      mergeMap(el => watchElementVisibility(el)\n        .pipe(\n          takeUntil(document$.pipe(skip(1))),\n          filter(visible => visible),\n          map(() => el),\n          take(1)\n        )\n      ),\n      filter(el => el.offsetWidth < el.scrollWidth),\n      mergeMap(el => {\n        const text = el.innerText\n        const host = el.closest(\"a\") || el\n        host.title = text\n\n        /* Mount tooltip */\n        return mountTooltip(host)\n          .pipe(\n            takeUntil(document$.pipe(skip(1))),\n            finalize(() => host.removeAttribute(\"title\"))\n          )\n      })\n    )\n      .subscribe()\n\n  // @todo move this outside of here and fix memleaks\n  document$\n    .pipe(\n      switchMap(() => getElements(\".md-status\")),\n      mergeMap(el => mountTooltip(el))\n    )\n      .subscribe()\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  fromEvent,\n  map,\n  mergeMap,\n  switchMap,\n  takeWhile,\n  tap,\n  withLatestFrom\n} from \"rxjs\"\n\nimport { getElements } from \"~/browser\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Patch options\n */\ninterface PatchOptions {\n  document$: Observable<Document>      /* Document observable */\n  tablet$: Observable<boolean>         /* Media tablet observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Patch indeterminate checkboxes\n *\n * This function replaces the indeterminate \"pseudo state\" with the actual\n * indeterminate state, which is used to keep navigation always expanded.\n *\n * @param options - Options\n */\nexport function patchIndeterminate(\n  { document$, tablet$ }: PatchOptions\n): void {\n  document$\n    .pipe(\n      switchMap(() => getElements<HTMLInputElement>(\n        \".md-toggle--indeterminate\"\n      )),\n      tap(el => {\n        el.indeterminate = true\n        el.checked = false\n      }),\n      mergeMap(el => fromEvent(el, \"change\")\n        .pipe(\n          takeWhile(() => el.classList.contains(\"md-toggle--indeterminate\")),\n          map(() => el)\n        )\n      ),\n      withLatestFrom(tablet$)\n    )\n      .subscribe(([el, tablet]) => {\n        el.classList.remove(\"md-toggle--indeterminate\")\n        if (tablet)\n          el.checked = false\n      })\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  filter,\n  fromEvent,\n  map,\n  mergeMap,\n  switchMap,\n  tap\n} from \"rxjs\"\n\nimport { getElements } from \"~/browser\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Patch options\n */\ninterface PatchOptions {\n  document$: Observable<Document>      /* Document observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Check whether the given device is an Apple device\n *\n * @returns Test result\n */\nfunction isAppleDevice(): boolean {\n  return /(iPad|iPhone|iPod)/.test(navigator.userAgent)\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Patch all elements with `data-md-scrollfix` attributes\n *\n * This is a year-old patch which ensures that overflow scrolling works at the\n * top and bottom of containers on iOS by ensuring a `1px` scroll offset upon\n * the start of a touch event.\n *\n * @see https://bit.ly/2SCtAOO - Original source\n *\n * @param options - Options\n */\nexport function patchScrollfix(\n  { document$ }: PatchOptions\n): void {\n  document$\n    .pipe(\n      switchMap(() => getElements(\"[data-md-scrollfix]\")),\n      tap(el => el.removeAttribute(\"data-md-scrollfix\")),\n      filter(isAppleDevice),\n      mergeMap(el => fromEvent(el, \"touchstart\")\n        .pipe(\n          map(() => el)\n        )\n      )\n    )\n      .subscribe(el => {\n        const top = el.scrollTop\n\n        /* We're at the top of the container */\n        if (top === 0) {\n          el.scrollTop = 1\n\n        /* We're at the bottom of the container */\n        } else if (top + el.offsetHeight === el.scrollHeight) {\n          el.scrollTop = top - 1\n        }\n      })\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  combineLatest,\n  delay,\n  map,\n  of,\n  switchMap,\n  withLatestFrom\n} from \"rxjs\"\n\nimport {\n  Viewport,\n  watchToggle\n} from \"~/browser\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Patch options\n */\ninterface PatchOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  tablet$: Observable<boolean>         /* Media tablet observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Patch the document body to lock when search is open\n *\n * For mobile and tablet viewports, the search is rendered full screen, which\n * leads to scroll leaking when at the top or bottom of the search result. This\n * function locks the body when the search is in full screen mode, and restores\n * the scroll position when leaving.\n *\n * @param options - Options\n */\nexport function patchScrolllock(\n  { viewport$, tablet$ }: PatchOptions\n): void {\n  combineLatest([watchToggle(\"search\"), tablet$])\n    .pipe(\n      map(([active, tablet]) => active && !tablet),\n      switchMap(active => of(active)\n        .pipe(\n          delay(active ? 400 : 100)\n        )\n      ),\n      withLatestFrom(viewport$)\n    )\n      .subscribe(([active, { offset: { y }}]) => {\n        if (active) {\n          document.body.setAttribute(\"data-md-scrolllock\", \"\")\n          document.body.style.top = `-${y}px`\n        } else {\n          const value = -1 * parseInt(document.body.style.top, 10)\n          document.body.removeAttribute(\"data-md-scrolllock\")\n          document.body.style.top = \"\"\n          if (value)\n            window.scrollTo(0, value)\n        }\n      })\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Polyfills\n * ------------------------------------------------------------------------- */\n\n/* Polyfill `Object.entries` */\nif (!Object.entries)\n  Object.entries = function (obj: object) {\n    const data: [string, string][] = []\n    for (const key of Object.keys(obj))\n      // @ts-expect-error - ignore property access warning\n      data.push([key, obj[key]])\n\n    /* Return entries */\n    return data\n  }\n\n/* Polyfill `Object.values` */\nif (!Object.values)\n  Object.values = function (obj: object) {\n    const data: string[] = []\n    for (const key of Object.keys(obj))\n      // @ts-expect-error - ignore property access warning\n      data.push(obj[key])\n\n    /* Return values */\n    return data\n  }\n\n/* ------------------------------------------------------------------------- */\n\n/* Polyfills for `Element` */\nif (typeof Element !== \"undefined\") {\n\n  /* Polyfill `Element.scrollTo` */\n  if (!Element.prototype.scrollTo)\n    Element.prototype.scrollTo = function (\n      x?: ScrollToOptions | number, y?: number\n    ): void {\n      if (typeof x === \"object\") {\n        this.scrollLeft = x.left!\n        this.scrollTop = x.top!\n      } else {\n        this.scrollLeft = x!\n        this.scrollTop = y!\n      }\n    }\n\n  /* Polyfill `Element.replaceWith` */\n  if (!Element.prototype.replaceWith)\n    Element.prototype.replaceWith = function (\n      ...nodes: Array<string | Node>\n    ): void {\n      const parent = this.parentNode\n      if (parent) {\n        if (nodes.length === 0)\n          parent.removeChild(this)\n\n        /* Replace children and create text nodes */\n        for (let i = nodes.length - 1; i >= 0; i--) {\n          let node = nodes[i]\n          if (typeof node === \"string\")\n            node = document.createTextNode(node)\n          else if (node.parentNode)\n            node.parentNode.removeChild(node)\n\n          /* Replace child or insert before previous sibling */\n          if (!i)\n            parent.replaceChild(node, this)\n          else\n            parent.insertBefore(this.previousSibling!, node)\n        }\n      }\n    }\n}\n"],
+  "mappings": "2rCAAA,IAAAA,GAAAC,GAAA,CAAAC,GAAAC,KAAA,EAAC,SAAUC,EAAQC,EAAS,CAC1B,OAAOH,IAAY,UAAY,OAAOC,IAAW,YAAcE,EAAQ,EACvE,OAAO,QAAW,YAAc,OAAO,IAAM,OAAOA,CAAO,EAC1DA,EAAQ,CACX,GAAEH,GAAO,UAAY,CAAE,aASrB,SAASI,EAA0BC,EAAO,CACxC,IAAIC,EAAmB,GACnBC,EAA0B,GAC1BC,EAAiC,KAEjCC,EAAsB,CACxB,KAAM,GACN,OAAQ,GACR,IAAK,GACL,IAAK,GACL,MAAO,GACP,SAAU,GACV,OAAQ,GACR,KAAM,GACN,MAAO,GACP,KAAM,GACN,KAAM,GACN,SAAU,GACV,iBAAkB,EACpB,EAOA,SAASC,EAAmBC,EAAI,CAC9B,MACE,GAAAA,GACAA,IAAO,UACPA,EAAG,WAAa,QAChBA,EAAG,WAAa,QAChB,cAAeA,GACf,aAAcA,EAAG,UAKrB,CASA,SAASC,EAA8BD,EAAI,CACzC,IAAIE,GAAOF,EAAG,KACVG,GAAUH,EAAG,QAUjB,MARI,GAAAG,KAAY,SAAWL,EAAoBI,EAAI,GAAK,CAACF,EAAG,UAIxDG,KAAY,YAAc,CAACH,EAAG,UAI9BA,EAAG,kBAKT,CAOA,SAASI,EAAqBJ,EAAI,CAC5BA,EAAG,UAAU,SAAS,eAAe,IAGzCA,EAAG,UAAU,IAAI,eAAe,EAChCA,EAAG,aAAa,2BAA4B,EAAE,EAChD,CAOA,SAASK,EAAwBL,EAAI,CAC9BA,EAAG,aAAa,0BAA0B,IAG/CA,EAAG,UAAU,OAAO,eAAe,EACnCA,EAAG,gBAAgB,0BAA0B,EAC/C,CAUA,SAASM,EAAUC,EAAG,CAChBA,EAAE,SAAWA,EAAE,QAAUA,EAAE,UAI3BR,EAAmBL,EAAM,aAAa,GACxCU,EAAqBV,EAAM,aAAa,EAG1CC,EAAmB,GACrB,CAUA,SAASa,EAAcD,EAAG,CACxBZ,EAAmB,EACrB,CASA,SAASc,EAAQF,EAAG,CAEbR,EAAmBQ,EAAE,MAAM,IAI5BZ,GAAoBM,EAA8BM,EAAE,MAAM,IAC5DH,EAAqBG,EAAE,MAAM,CAEjC,CAMA,SAASG,EAAOH,EAAG,CACZR,EAAmBQ,EAAE,MAAM,IAK9BA,EAAE,OAAO,UAAU,SAAS,eAAe,GAC3CA,EAAE,OAAO,aAAa,0BAA0B,KAMhDX,EAA0B,GAC1B,OAAO,aAAaC,CAA8B,EAClDA,EAAiC,OAAO,WAAW,UAAW,CAC5DD,EAA0B,EAC5B,EAAG,GAAG,EACNS,EAAwBE,EAAE,MAAM,EAEpC,CAOA,SAASI,EAAmBJ,EAAG,CACzB,SAAS,kBAAoB,WAK3BX,IACFD,EAAmB,IAErBiB,EAA+B,EAEnC,CAQA,SAASA,GAAiC,CACxC,SAAS,iBAAiB,YAAaC,CAAoB,EAC3D,SAAS,iBAAiB,YAAaA,CAAoB,EAC3D,SAAS,iBAAiB,UAAWA,CAAoB,EACzD,SAAS,iBAAiB,cAAeA,CAAoB,EAC7D,SAAS,iBAAiB,cAAeA,CAAoB,EAC7D,SAAS,iBAAiB,YAAaA,CAAoB,EAC3D,SAAS,iBAAiB,YAAaA,CAAoB,EAC3D,SAAS,iBAAiB,aAAcA,CAAoB,EAC5D,SAAS,iBAAiB,WAAYA,CAAoB,CAC5D,CAEA,SAASC,IAAoC,CAC3C,SAAS,oBAAoB,YAAaD,CAAoB,EAC9D,SAAS,oBAAoB,YAAaA,CAAoB,EAC9D,SAAS,oBAAoB,UAAWA,CAAoB,EAC5D,SAAS,oBAAoB,cAAeA,CAAoB,EAChE,SAAS,oBAAoB,cAAeA,CAAoB,EAChE,SAAS,oBAAoB,YAAaA,CAAoB,EAC9D,SAAS,oBAAoB,YAAaA,CAAoB,EAC9D,SAAS,oBAAoB,aAAcA,CAAoB,EAC/D,SAAS,oBAAoB,WAAYA,CAAoB,CAC/D,CASA,SAASA,EAAqBN,EAAG,CAG3BA,EAAE,OAAO,UAAYA,EAAE,OAAO,SAAS,YAAY,IAAM,SAI7DZ,EAAmB,GACnBmB,GAAkC,EACpC,CAKA,SAAS,iBAAiB,UAAWR,EAAW,EAAI,EACpD,SAAS,iBAAiB,YAAaE,EAAe,EAAI,EAC1D,SAAS,iBAAiB,cAAeA,EAAe,EAAI,EAC5D,SAAS,iBAAiB,aAAcA,EAAe,EAAI,EAC3D,SAAS,iBAAiB,mBAAoBG,EAAoB,EAAI,EAEtEC,EAA+B,EAM/BlB,EAAM,iBAAiB,QAASe,EAAS,EAAI,EAC7Cf,EAAM,iBAAiB,OAAQgB,EAAQ,EAAI,EAOvChB,EAAM,WAAa,KAAK,wBAA0BA,EAAM,KAI1DA,EAAM,KAAK,aAAa,wBAAyB,EAAE,EAC1CA,EAAM,WAAa,KAAK,gBACjC,SAAS,gBAAgB,UAAU,IAAI,kBAAkB,EACzD,SAAS,gBAAgB,aAAa,wBAAyB,EAAE,EAErE,CAKA,GAAI,OAAO,QAAW,aAAe,OAAO,UAAa,YAAa,CAIpE,OAAO,0BAA4BD,EAInC,IAAIsB,EAEJ,GAAI,CACFA,EAAQ,IAAI,YAAY,8BAA8B,CACxD,OAASC,EAAO,CAEdD,EAAQ,SAAS,YAAY,aAAa,EAC1CA,EAAM,gBAAgB,+BAAgC,GAAO,GAAO,CAAC,CAAC,CACxE,CAEA,OAAO,cAAcA,CAAK,CAC5B,CAEI,OAAO,UAAa,aAGtBtB,EAA0B,QAAQ,CAGtC,CAAE,ICvTF,IAAAwB,GAAAC,GAAA,CAAAC,GAAAC,KAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAMC,SAA0CC,EAAMC,EAAS,CACtD,OAAOH,IAAY,UAAY,OAAOC,IAAW,SACnDA,GAAO,QAAUE,EAAQ,EAClB,OAAO,QAAW,YAAc,OAAO,IAC9C,OAAO,CAAC,EAAGA,CAAO,EACX,OAAOH,IAAY,SAC1BA,GAAQ,YAAiBG,EAAQ,EAEjCD,EAAK,YAAiBC,EAAQ,CAChC,GAAGH,GAAM,UAAW,CACpB,OAAiB,UAAW,CAClB,IAAII,EAAuB,CAE/B,IACC,SAASC,EAAyBC,EAAqBC,EAAqB,CAEnF,aAGAA,EAAoB,EAAED,EAAqB,CACzC,QAAW,UAAW,CAAE,OAAqBE,EAAW,CAC1D,CAAC,EAGD,IAAIC,EAAeF,EAAoB,GAAG,EACtCG,EAAoCH,EAAoB,EAAEE,CAAY,EAEtEE,EAASJ,EAAoB,GAAG,EAChCK,EAA8BL,EAAoB,EAAEI,CAAM,EAE1DE,EAAaN,EAAoB,GAAG,EACpCO,EAA8BP,EAAoB,EAAEM,CAAU,EAOlE,SAASE,EAAQC,EAAM,CACrB,GAAI,CACF,OAAO,SAAS,YAAYA,CAAI,CAClC,OAASC,EAAK,CACZ,MAAO,EACT,CACF,CAUA,IAAIC,EAAqB,SAA4BC,EAAQ,CAC3D,IAAIC,EAAeN,EAAe,EAAEK,CAAM,EAC1C,OAAAJ,EAAQ,KAAK,EACNK,CACT,EAEiCC,EAAeH,EAOhD,SAASI,EAAkBC,EAAO,CAChC,IAAIC,EAAQ,SAAS,gBAAgB,aAAa,KAAK,IAAM,MACzDC,EAAc,SAAS,cAAc,UAAU,EAEnDA,EAAY,MAAM,SAAW,OAE7BA,EAAY,MAAM,OAAS,IAC3BA,EAAY,MAAM,QAAU,IAC5BA,EAAY,MAAM,OAAS,IAE3BA,EAAY,MAAM,SAAW,WAC7BA,EAAY,MAAMD,EAAQ,QAAU,MAAM,EAAI,UAE9C,IAAIE,EAAY,OAAO,aAAe,SAAS,gBAAgB,UAC/D,OAAAD,EAAY,MAAM,IAAM,GAAG,OAAOC,EAAW,IAAI,EACjDD,EAAY,aAAa,WAAY,EAAE,EACvCA,EAAY,MAAQF,EACbE,CACT,CAYA,IAAIE,EAAiB,SAAwBJ,EAAOK,EAAS,CAC3D,IAAIH,EAAcH,EAAkBC,CAAK,EACzCK,EAAQ,UAAU,YAAYH,CAAW,EACzC,IAAIL,EAAeN,EAAe,EAAEW,CAAW,EAC/C,OAAAV,EAAQ,MAAM,EACdU,EAAY,OAAO,EACZL,CACT,EASIS,GAAsB,SAA6BV,EAAQ,CAC7D,IAAIS,EAAU,UAAU,OAAS,GAAK,UAAU,CAAC,IAAM,OAAY,UAAU,CAAC,EAAI,CAChF,UAAW,SAAS,IACtB,EACIR,EAAe,GAEnB,OAAI,OAAOD,GAAW,SACpBC,EAAeO,EAAeR,EAAQS,CAAO,EACpCT,aAAkB,kBAAoB,CAAC,CAAC,OAAQ,SAAU,MAAO,MAAO,UAAU,EAAE,SAASA,GAAW,KAA4B,OAASA,EAAO,IAAI,EAEjKC,EAAeO,EAAeR,EAAO,MAAOS,CAAO,GAEnDR,EAAeN,EAAe,EAAEK,CAAM,EACtCJ,EAAQ,MAAM,GAGTK,CACT,EAEiCU,EAAgBD,GAEjD,SAASE,EAAQC,EAAK,CAAE,0BAA2B,OAAI,OAAO,QAAW,YAAc,OAAO,OAAO,UAAa,SAAYD,EAAU,SAAiBC,EAAK,CAAE,OAAO,OAAOA,CAAK,EAAYD,EAAU,SAAiBC,EAAK,CAAE,OAAOA,GAAO,OAAO,QAAW,YAAcA,EAAI,cAAgB,QAAUA,IAAQ,OAAO,UAAY,SAAW,OAAOA,CAAK,EAAYD,EAAQC,CAAG,CAAG,CAUzX,IAAIC,GAAyB,UAAkC,CAC7D,IAAIL,EAAU,UAAU,OAAS,GAAK,UAAU,CAAC,IAAM,OAAY,UAAU,CAAC,EAAI,CAAC,EAE/EM,EAAkBN,EAAQ,OAC1BO,EAASD,IAAoB,OAAS,OAASA,EAC/CE,EAAYR,EAAQ,UACpBT,EAASS,EAAQ,OACjBS,GAAOT,EAAQ,KAEnB,GAAIO,IAAW,QAAUA,IAAW,MAClC,MAAM,IAAI,MAAM,oDAAoD,EAItE,GAAIhB,IAAW,OACb,GAAIA,GAAUY,EAAQZ,CAAM,IAAM,UAAYA,EAAO,WAAa,EAAG,CACnE,GAAIgB,IAAW,QAAUhB,EAAO,aAAa,UAAU,EACrD,MAAM,IAAI,MAAM,mFAAmF,EAGrG,GAAIgB,IAAW,QAAUhB,EAAO,aAAa,UAAU,GAAKA,EAAO,aAAa,UAAU,GACxF,MAAM,IAAI,MAAM,uGAAwG,CAE5H,KACE,OAAM,IAAI,MAAM,6CAA6C,EAKjE,GAAIkB,GACF,OAAOP,EAAaO,GAAM,CACxB,UAAWD,CACb,CAAC,EAIH,GAAIjB,EACF,OAAOgB,IAAW,MAAQd,EAAYF,CAAM,EAAIW,EAAaX,EAAQ,CACnE,UAAWiB,CACb,CAAC,CAEL,EAEiCE,GAAmBL,GAEpD,SAASM,GAAiBP,EAAK,CAAE,0BAA2B,OAAI,OAAO,QAAW,YAAc,OAAO,OAAO,UAAa,SAAYO,GAAmB,SAAiBP,EAAK,CAAE,OAAO,OAAOA,CAAK,EAAYO,GAAmB,SAAiBP,EAAK,CAAE,OAAOA,GAAO,OAAO,QAAW,YAAcA,EAAI,cAAgB,QAAUA,IAAQ,OAAO,UAAY,SAAW,OAAOA,CAAK,EAAYO,GAAiBP,CAAG,CAAG,CAE7Z,SAASQ,GAAgBC,EAAUC,EAAa,CAAE,GAAI,EAAED,aAAoBC,GAAgB,MAAM,IAAI,UAAU,mCAAmC,CAAK,CAExJ,SAASC,GAAkBxB,EAAQyB,EAAO,CAAE,QAASC,EAAI,EAAGA,EAAID,EAAM,OAAQC,IAAK,CAAE,IAAIC,EAAaF,EAAMC,CAAC,EAAGC,EAAW,WAAaA,EAAW,YAAc,GAAOA,EAAW,aAAe,GAAU,UAAWA,IAAYA,EAAW,SAAW,IAAM,OAAO,eAAe3B,EAAQ2B,EAAW,IAAKA,CAAU,CAAG,CAAE,CAE5T,SAASC,GAAaL,EAAaM,EAAYC,EAAa,CAAE,OAAID,GAAYL,GAAkBD,EAAY,UAAWM,CAAU,EAAOC,GAAaN,GAAkBD,EAAaO,CAAW,EAAUP,CAAa,CAEtN,SAASQ,GAAUC,EAAUC,EAAY,CAAE,GAAI,OAAOA,GAAe,YAAcA,IAAe,KAAQ,MAAM,IAAI,UAAU,oDAAoD,EAAKD,EAAS,UAAY,OAAO,OAAOC,GAAcA,EAAW,UAAW,CAAE,YAAa,CAAE,MAAOD,EAAU,SAAU,GAAM,aAAc,EAAK,CAAE,CAAC,EAAOC,GAAYC,GAAgBF,EAAUC,CAAU,CAAG,CAEhY,SAASC,GAAgBC,EAAGC,EAAG,CAAE,OAAAF,GAAkB,OAAO,gBAAkB,SAAyBC,EAAGC,EAAG,CAAE,OAAAD,EAAE,UAAYC,EAAUD,CAAG,EAAUD,GAAgBC,EAAGC,CAAC,CAAG,CAEzK,SAASC,GAAaC,EAAS,CAAE,IAAIC,EAA4BC,GAA0B,EAAG,OAAO,UAAgC,CAAE,IAAIC,EAAQC,GAAgBJ,CAAO,EAAGK,EAAQ,GAAIJ,EAA2B,CAAE,IAAIK,EAAYF,GAAgB,IAAI,EAAE,YAAaC,EAAS,QAAQ,UAAUF,EAAO,UAAWG,CAAS,CAAG,MAASD,EAASF,EAAM,MAAM,KAAM,SAAS,EAAK,OAAOI,GAA2B,KAAMF,CAAM,CAAG,CAAG,CAExa,SAASE,GAA2BC,EAAMC,EAAM,CAAE,OAAIA,IAAS3B,GAAiB2B,CAAI,IAAM,UAAY,OAAOA,GAAS,YAAsBA,EAAeC,GAAuBF,CAAI,CAAG,CAEzL,SAASE,GAAuBF,EAAM,CAAE,GAAIA,IAAS,OAAU,MAAM,IAAI,eAAe,2DAA2D,EAAK,OAAOA,CAAM,CAErK,SAASN,IAA4B,CAA0E,GAApE,OAAO,SAAY,aAAe,CAAC,QAAQ,WAA6B,QAAQ,UAAU,KAAM,MAAO,GAAO,GAAI,OAAO,OAAU,WAAY,MAAO,GAAM,GAAI,CAAE,YAAK,UAAU,SAAS,KAAK,QAAQ,UAAU,KAAM,CAAC,EAAG,UAAY,CAAC,CAAC,CAAC,EAAU,EAAM,OAASS,EAAG,CAAE,MAAO,EAAO,CAAE,CAEnU,SAASP,GAAgBP,EAAG,CAAE,OAAAO,GAAkB,OAAO,eAAiB,OAAO,eAAiB,SAAyBP,EAAG,CAAE,OAAOA,EAAE,WAAa,OAAO,eAAeA,CAAC,CAAG,EAAUO,GAAgBP,CAAC,CAAG,CAa5M,SAASe,GAAkBC,EAAQC,EAAS,CAC1C,IAAIC,EAAY,kBAAkB,OAAOF,CAAM,EAE/C,GAAKC,EAAQ,aAAaC,CAAS,EAInC,OAAOD,EAAQ,aAAaC,CAAS,CACvC,CAOA,IAAIC,GAAyB,SAAUC,EAAU,CAC/CxB,GAAUuB,EAAWC,CAAQ,EAE7B,IAAIC,EAASnB,GAAaiB,CAAS,EAMnC,SAASA,EAAUG,EAAShD,EAAS,CACnC,IAAIiD,EAEJ,OAAArC,GAAgB,KAAMiC,CAAS,EAE/BI,EAAQF,EAAO,KAAK,IAAI,EAExBE,EAAM,eAAejD,CAAO,EAE5BiD,EAAM,YAAYD,CAAO,EAElBC,CACT,CAQA,OAAA9B,GAAa0B,EAAW,CAAC,CACvB,IAAK,iBACL,MAAO,UAA0B,CAC/B,IAAI7C,EAAU,UAAU,OAAS,GAAK,UAAU,CAAC,IAAM,OAAY,UAAU,CAAC,EAAI,CAAC,EACnF,KAAK,OAAS,OAAOA,EAAQ,QAAW,WAAaA,EAAQ,OAAS,KAAK,cAC3E,KAAK,OAAS,OAAOA,EAAQ,QAAW,WAAaA,EAAQ,OAAS,KAAK,cAC3E,KAAK,KAAO,OAAOA,EAAQ,MAAS,WAAaA,EAAQ,KAAO,KAAK,YACrE,KAAK,UAAYW,GAAiBX,EAAQ,SAAS,IAAM,SAAWA,EAAQ,UAAY,SAAS,IACnG,CAMF,EAAG,CACD,IAAK,cACL,MAAO,SAAqBgD,EAAS,CACnC,IAAIE,EAAS,KAEb,KAAK,SAAWlE,EAAe,EAAEgE,EAAS,QAAS,SAAUR,GAAG,CAC9D,OAAOU,EAAO,QAAQV,EAAC,CACzB,CAAC,CACH,CAMF,EAAG,CACD,IAAK,UACL,MAAO,SAAiBA,EAAG,CACzB,IAAIQ,EAAUR,EAAE,gBAAkBA,EAAE,cAChCjC,GAAS,KAAK,OAAOyC,CAAO,GAAK,OACjCvC,GAAOC,GAAgB,CACzB,OAAQH,GACR,UAAW,KAAK,UAChB,OAAQ,KAAK,OAAOyC,CAAO,EAC3B,KAAM,KAAK,KAAKA,CAAO,CACzB,CAAC,EAED,KAAK,KAAKvC,GAAO,UAAY,QAAS,CACpC,OAAQF,GACR,KAAME,GACN,QAASuC,EACT,eAAgB,UAA0B,CACpCA,GACFA,EAAQ,MAAM,EAGhB,OAAO,aAAa,EAAE,gBAAgB,CACxC,CACF,CAAC,CACH,CAMF,EAAG,CACD,IAAK,gBACL,MAAO,SAAuBA,EAAS,CACrC,OAAOP,GAAkB,SAAUO,CAAO,CAC5C,CAMF,EAAG,CACD,IAAK,gBACL,MAAO,SAAuBA,EAAS,CACrC,IAAIG,EAAWV,GAAkB,SAAUO,CAAO,EAElD,GAAIG,EACF,OAAO,SAAS,cAAcA,CAAQ,CAE1C,CAQF,EAAG,CACD,IAAK,cAML,MAAO,SAAqBH,EAAS,CACnC,OAAOP,GAAkB,OAAQO,CAAO,CAC1C,CAKF,EAAG,CACD,IAAK,UACL,MAAO,UAAmB,CACxB,KAAK,SAAS,QAAQ,CACxB,CACF,CAAC,EAAG,CAAC,CACH,IAAK,OACL,MAAO,SAAczD,EAAQ,CAC3B,IAAIS,EAAU,UAAU,OAAS,GAAK,UAAU,CAAC,IAAM,OAAY,UAAU,CAAC,EAAI,CAChF,UAAW,SAAS,IACtB,EACA,OAAOE,EAAaX,EAAQS,CAAO,CACrC,CAOF,EAAG,CACD,IAAK,MACL,MAAO,SAAaT,EAAQ,CAC1B,OAAOE,EAAYF,CAAM,CAC3B,CAOF,EAAG,CACD,IAAK,cACL,MAAO,UAAuB,CAC5B,IAAIgB,EAAS,UAAU,OAAS,GAAK,UAAU,CAAC,IAAM,OAAY,UAAU,CAAC,EAAI,CAAC,OAAQ,KAAK,EAC3F6C,EAAU,OAAO7C,GAAW,SAAW,CAACA,CAAM,EAAIA,EAClD8C,GAAU,CAAC,CAAC,SAAS,sBACzB,OAAAD,EAAQ,QAAQ,SAAU7C,GAAQ,CAChC8C,GAAUA,IAAW,CAAC,CAAC,SAAS,sBAAsB9C,EAAM,CAC9D,CAAC,EACM8C,EACT,CACF,CAAC,CAAC,EAEKR,CACT,EAAG/D,EAAqB,CAAE,EAEOF,GAAaiE,EAExC,EAEA,IACC,SAASxE,EAAQ,CAExB,IAAIiF,EAAqB,EAKzB,GAAI,OAAO,SAAY,aAAe,CAAC,QAAQ,UAAU,QAAS,CAC9D,IAAIC,EAAQ,QAAQ,UAEpBA,EAAM,QAAUA,EAAM,iBACNA,EAAM,oBACNA,EAAM,mBACNA,EAAM,kBACNA,EAAM,qBAC1B,CASA,SAASC,EAASb,EAASQ,EAAU,CACjC,KAAOR,GAAWA,EAAQ,WAAaW,GAAoB,CACvD,GAAI,OAAOX,EAAQ,SAAY,YAC3BA,EAAQ,QAAQQ,CAAQ,EAC1B,OAAOR,EAETA,EAAUA,EAAQ,UACtB,CACJ,CAEAtE,EAAO,QAAUmF,CAGX,EAEA,IACC,SAASnF,EAAQoF,EAA0B9E,EAAqB,CAEvE,IAAI6E,EAAU7E,EAAoB,GAAG,EAYrC,SAAS+E,EAAUf,EAASQ,EAAU/D,EAAMuE,EAAUC,EAAY,CAC9D,IAAIC,EAAaC,EAAS,MAAM,KAAM,SAAS,EAE/C,OAAAnB,EAAQ,iBAAiBvD,EAAMyE,EAAYD,CAAU,EAE9C,CACH,QAAS,UAAW,CAChBjB,EAAQ,oBAAoBvD,EAAMyE,EAAYD,CAAU,CAC5D,CACJ,CACJ,CAYA,SAASG,EAASC,EAAUb,EAAU/D,EAAMuE,EAAUC,EAAY,CAE9D,OAAI,OAAOI,EAAS,kBAAqB,WAC9BN,EAAU,MAAM,KAAM,SAAS,EAItC,OAAOtE,GAAS,WAGTsE,EAAU,KAAK,KAAM,QAAQ,EAAE,MAAM,KAAM,SAAS,GAI3D,OAAOM,GAAa,WACpBA,EAAW,SAAS,iBAAiBA,CAAQ,GAI1C,MAAM,UAAU,IAAI,KAAKA,EAAU,SAAUrB,EAAS,CACzD,OAAOe,EAAUf,EAASQ,EAAU/D,EAAMuE,EAAUC,CAAU,CAClE,CAAC,EACL,CAWA,SAASE,EAASnB,EAASQ,EAAU/D,EAAMuE,EAAU,CACjD,OAAO,SAASnB,EAAG,CACfA,EAAE,eAAiBgB,EAAQhB,EAAE,OAAQW,CAAQ,EAEzCX,EAAE,gBACFmB,EAAS,KAAKhB,EAASH,CAAC,CAEhC,CACJ,CAEAnE,EAAO,QAAU0F,CAGX,EAEA,IACC,SAAStF,EAAyBL,EAAS,CAQlDA,EAAQ,KAAO,SAASuB,EAAO,CAC3B,OAAOA,IAAU,QACVA,aAAiB,aACjBA,EAAM,WAAa,CAC9B,EAQAvB,EAAQ,SAAW,SAASuB,EAAO,CAC/B,IAAIP,EAAO,OAAO,UAAU,SAAS,KAAKO,CAAK,EAE/C,OAAOA,IAAU,SACTP,IAAS,qBAAuBA,IAAS,4BACzC,WAAYO,IACZA,EAAM,SAAW,GAAKvB,EAAQ,KAAKuB,EAAM,CAAC,CAAC,EACvD,EAQAvB,EAAQ,OAAS,SAASuB,EAAO,CAC7B,OAAO,OAAOA,GAAU,UACjBA,aAAiB,MAC5B,EAQAvB,EAAQ,GAAK,SAASuB,EAAO,CACzB,IAAIP,EAAO,OAAO,UAAU,SAAS,KAAKO,CAAK,EAE/C,OAAOP,IAAS,mBACpB,CAGM,EAEA,IACC,SAASf,EAAQoF,EAA0B9E,EAAqB,CAEvE,IAAIsF,EAAKtF,EAAoB,GAAG,EAC5BoF,EAAWpF,EAAoB,GAAG,EAWtC,SAASI,EAAOQ,EAAQH,EAAMuE,EAAU,CACpC,GAAI,CAACpE,GAAU,CAACH,GAAQ,CAACuE,EACrB,MAAM,IAAI,MAAM,4BAA4B,EAGhD,GAAI,CAACM,EAAG,OAAO7E,CAAI,EACf,MAAM,IAAI,UAAU,kCAAkC,EAG1D,GAAI,CAAC6E,EAAG,GAAGN,CAAQ,EACf,MAAM,IAAI,UAAU,mCAAmC,EAG3D,GAAIM,EAAG,KAAK1E,CAAM,EACd,OAAO2E,EAAW3E,EAAQH,EAAMuE,CAAQ,EAEvC,GAAIM,EAAG,SAAS1E,CAAM,EACvB,OAAO4E,EAAe5E,EAAQH,EAAMuE,CAAQ,EAE3C,GAAIM,EAAG,OAAO1E,CAAM,EACrB,OAAO6E,EAAe7E,EAAQH,EAAMuE,CAAQ,EAG5C,MAAM,IAAI,UAAU,2EAA2E,CAEvG,CAWA,SAASO,EAAWG,EAAMjF,EAAMuE,EAAU,CACtC,OAAAU,EAAK,iBAAiBjF,EAAMuE,CAAQ,EAE7B,CACH,QAAS,UAAW,CAChBU,EAAK,oBAAoBjF,EAAMuE,CAAQ,CAC3C,CACJ,CACJ,CAWA,SAASQ,EAAeG,EAAUlF,EAAMuE,EAAU,CAC9C,aAAM,UAAU,QAAQ,KAAKW,EAAU,SAASD,EAAM,CAClDA,EAAK,iBAAiBjF,EAAMuE,CAAQ,CACxC,CAAC,EAEM,CACH,QAAS,UAAW,CAChB,MAAM,UAAU,QAAQ,KAAKW,EAAU,SAASD,EAAM,CAClDA,EAAK,oBAAoBjF,EAAMuE,CAAQ,CAC3C,CAAC,CACL,CACJ,CACJ,CAWA,SAASS,EAAejB,EAAU/D,EAAMuE,EAAU,CAC9C,OAAOI,EAAS,SAAS,KAAMZ,EAAU/D,EAAMuE,CAAQ,CAC3D,CAEAtF,EAAO,QAAUU,CAGX,EAEA,IACC,SAASV,EAAQ,CAExB,SAASkG,EAAO5B,EAAS,CACrB,IAAInD,EAEJ,GAAImD,EAAQ,WAAa,SACrBA,EAAQ,MAAM,EAEdnD,EAAemD,EAAQ,cAElBA,EAAQ,WAAa,SAAWA,EAAQ,WAAa,WAAY,CACtE,IAAI6B,EAAa7B,EAAQ,aAAa,UAAU,EAE3C6B,GACD7B,EAAQ,aAAa,WAAY,EAAE,EAGvCA,EAAQ,OAAO,EACfA,EAAQ,kBAAkB,EAAGA,EAAQ,MAAM,MAAM,EAE5C6B,GACD7B,EAAQ,gBAAgB,UAAU,EAGtCnD,EAAemD,EAAQ,KAC3B,KACK,CACGA,EAAQ,aAAa,iBAAiB,GACtCA,EAAQ,MAAM,EAGlB,IAAI8B,EAAY,OAAO,aAAa,EAChCC,EAAQ,SAAS,YAAY,EAEjCA,EAAM,mBAAmB/B,CAAO,EAChC8B,EAAU,gBAAgB,EAC1BA,EAAU,SAASC,CAAK,EAExBlF,EAAeiF,EAAU,SAAS,CACtC,CAEA,OAAOjF,CACX,CAEAnB,EAAO,QAAUkG,CAGX,EAEA,IACC,SAASlG,EAAQ,CAExB,SAASsG,GAAK,CAGd,CAEAA,EAAE,UAAY,CACZ,GAAI,SAAUC,EAAMjB,EAAUkB,EAAK,CACjC,IAAIrC,EAAI,KAAK,IAAM,KAAK,EAAI,CAAC,GAE7B,OAACA,EAAEoC,CAAI,IAAMpC,EAAEoC,CAAI,EAAI,CAAC,IAAI,KAAK,CAC/B,GAAIjB,EACJ,IAAKkB,CACP,CAAC,EAEM,IACT,EAEA,KAAM,SAAUD,EAAMjB,EAAUkB,EAAK,CACnC,IAAIxC,EAAO,KACX,SAASyB,GAAY,CACnBzB,EAAK,IAAIuC,EAAMd,CAAQ,EACvBH,EAAS,MAAMkB,EAAK,SAAS,CAC/B,CAEA,OAAAf,EAAS,EAAIH,EACN,KAAK,GAAGiB,EAAMd,EAAUe,CAAG,CACpC,EAEA,KAAM,SAAUD,EAAM,CACpB,IAAIE,EAAO,CAAC,EAAE,MAAM,KAAK,UAAW,CAAC,EACjCC,IAAW,KAAK,IAAM,KAAK,EAAI,CAAC,IAAIH,CAAI,GAAK,CAAC,GAAG,MAAM,EACvD3D,EAAI,EACJ+D,EAAMD,EAAO,OAEjB,IAAK9D,EAAGA,EAAI+D,EAAK/D,IACf8D,EAAO9D,CAAC,EAAE,GAAG,MAAM8D,EAAO9D,CAAC,EAAE,IAAK6D,CAAI,EAGxC,OAAO,IACT,EAEA,IAAK,SAAUF,EAAMjB,EAAU,CAC7B,IAAInB,EAAI,KAAK,IAAM,KAAK,EAAI,CAAC,GACzByC,EAAOzC,EAAEoC,CAAI,EACbM,EAAa,CAAC,EAElB,GAAID,GAAQtB,EACV,QAAS1C,EAAI,EAAG+D,EAAMC,EAAK,OAAQhE,EAAI+D,EAAK/D,IACtCgE,EAAKhE,CAAC,EAAE,KAAO0C,GAAYsB,EAAKhE,CAAC,EAAE,GAAG,IAAM0C,GAC9CuB,EAAW,KAAKD,EAAKhE,CAAC,CAAC,EAQ7B,OAACiE,EAAW,OACR1C,EAAEoC,CAAI,EAAIM,EACV,OAAO1C,EAAEoC,CAAI,EAEV,IACT,CACF,EAEAvG,EAAO,QAAUsG,EACjBtG,EAAO,QAAQ,YAAcsG,CAGvB,CAEI,EAGIQ,EAA2B,CAAC,EAGhC,SAASxG,EAAoByG,EAAU,CAEtC,GAAGD,EAAyBC,CAAQ,EACnC,OAAOD,EAAyBC,CAAQ,EAAE,QAG3C,IAAI/G,EAAS8G,EAAyBC,CAAQ,EAAI,CAGjD,QAAS,CAAC,CACX,EAGA,OAAA5G,EAAoB4G,CAAQ,EAAE/G,EAAQA,EAAO,QAASM,CAAmB,EAGlEN,EAAO,OACf,CAIA,OAAC,UAAW,CAEXM,EAAoB,EAAI,SAASN,EAAQ,CACxC,IAAIgH,EAAShH,GAAUA,EAAO,WAC7B,UAAW,CAAE,OAAOA,EAAO,OAAY,EACvC,UAAW,CAAE,OAAOA,CAAQ,EAC7B,OAAAM,EAAoB,EAAE0G,EAAQ,CAAE,EAAGA,CAAO,CAAC,EACpCA,CACR,CACD,EAAE,EAGD,UAAW,CAEX1G,EAAoB,EAAI,SAASP,EAASkH,EAAY,CACrD,QAAQC,KAAOD,EACX3G,EAAoB,EAAE2G,EAAYC,CAAG,GAAK,CAAC5G,EAAoB,EAAEP,EAASmH,CAAG,GAC/E,OAAO,eAAenH,EAASmH,EAAK,CAAE,WAAY,GAAM,IAAKD,EAAWC,CAAG,CAAE,CAAC,CAGjF,CACD,EAAE,EAGD,UAAW,CACX5G,EAAoB,EAAI,SAASyB,EAAKoF,EAAM,CAAE,OAAO,OAAO,UAAU,eAAe,KAAKpF,EAAKoF,CAAI,CAAG,CACvG,EAAE,EAMK7G,EAAoB,GAAG,CAC/B,EAAG,EACX,OACD,CAAC,ICz3BD,IAAA8G,GAAAC,GAAA,CAAAC,GAAAC,KAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAeA,IAAIC,GAAkB,UAOtBD,GAAO,QAAUE,GAUjB,SAASA,GAAWC,EAAQ,CAC1B,IAAIC,EAAM,GAAKD,EACXE,EAAQJ,GAAgB,KAAKG,CAAG,EAEpC,GAAI,CAACC,EACH,OAAOD,EAGT,IAAIE,EACAC,EAAO,GACPC,EAAQ,EACRC,EAAY,EAEhB,IAAKD,EAAQH,EAAM,MAAOG,EAAQJ,EAAI,OAAQI,IAAS,CACrD,OAAQJ,EAAI,WAAWI,CAAK,EAAG,CAC7B,IAAK,IACHF,EAAS,SACT,MACF,IAAK,IACHA,EAAS,QACT,MACF,IAAK,IACHA,EAAS,QACT,MACF,IAAK,IACHA,EAAS,OACT,MACF,IAAK,IACHA,EAAS,OACT,MACF,QACE,QACJ,CAEIG,IAAcD,IAChBD,GAAQH,EAAI,UAAUK,EAAWD,CAAK,GAGxCC,EAAYD,EAAQ,EACpBD,GAAQD,CACV,CAEA,OAAOG,IAAcD,EACjBD,EAAOH,EAAI,UAAUK,EAAWD,CAAK,EACrCD,CACN,ICvDA,IAAAG,GAAO,SCtBP;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,gFAgBA,IAAIC,GAAgB,SAASC,EAAGC,EAAG,CAC/B,OAAAF,GAAgB,OAAO,gBAClB,CAAE,UAAW,CAAC,CAAE,YAAa,OAAS,SAAUC,EAAGC,EAAG,CAAED,EAAE,UAAYC,CAAG,GAC1E,SAAUD,EAAGC,EAAG,CAAE,QAASC,KAAKD,EAAO,OAAO,UAAU,eAAe,KAAKA,EAAGC,CAAC,IAAGF,EAAEE,CAAC,EAAID,EAAEC,CAAC,EAAG,EAC7FH,GAAcC,EAAGC,CAAC,CAC7B,EAEO,SAASE,GAAUH,EAAGC,EAAG,CAC5B,GAAI,OAAOA,GAAM,YAAcA,IAAM,KACjC,MAAM,IAAI,UAAU,uBAAyB,OAAOA,CAAC,EAAI,+BAA+B,EAC5FF,GAAcC,EAAGC,CAAC,EAClB,SAASG,GAAK,CAAE,KAAK,YAAcJ,CAAG,CACtCA,EAAE,UAAYC,IAAM,KAAO,OAAO,OAAOA,CAAC,GAAKG,EAAG,UAAYH,EAAE,UAAW,IAAIG,EACnF,CAwCO,SAASC,GAAUC,EAASC,EAAYC,EAAGC,EAAW,CACzD,SAASC,EAAMC,EAAO,CAAE,OAAOA,aAAiBH,EAAIG,EAAQ,IAAIH,EAAE,SAAUI,EAAS,CAAEA,EAAQD,CAAK,CAAG,CAAC,CAAG,CAC3G,OAAO,IAAKH,IAAMA,EAAI,UAAU,SAAUI,EAASC,EAAQ,CACvD,SAASC,EAAUH,EAAO,CAAE,GAAI,CAAEI,EAAKN,EAAU,KAAKE,CAAK,CAAC,CAAG,OAASK,EAAG,CAAEH,EAAOG,CAAC,CAAG,CAAE,CAC1F,SAASC,EAASN,EAAO,CAAE,GAAI,CAAEI,EAAKN,EAAU,MAASE,CAAK,CAAC,CAAG,OAASK,EAAG,CAAEH,EAAOG,CAAC,CAAG,CAAE,CAC7F,SAASD,EAAKG,EAAQ,CAAEA,EAAO,KAAON,EAAQM,EAAO,KAAK,EAAIR,EAAMQ,EAAO,KAAK,EAAE,KAAKJ,EAAWG,CAAQ,CAAG,CAC7GF,GAAMN,EAAYA,EAAU,MAAMH,EAASC,GAAc,CAAC,CAAC,GAAG,KAAK,CAAC,CACxE,CAAC,CACL,CAEO,SAASY,GAAYb,EAASc,EAAM,CACvC,IAAIC,EAAI,CAAE,MAAO,EAAG,KAAM,UAAW,CAAE,GAAIC,EAAE,CAAC,EAAI,EAAG,MAAMA,EAAE,CAAC,EAAG,OAAOA,EAAE,CAAC,CAAG,EAAG,KAAM,CAAC,EAAG,IAAK,CAAC,CAAE,EAAGC,EAAGC,EAAGF,EAAGG,EAC/G,OAAOA,EAAI,CAAE,KAAMC,EAAK,CAAC,EAAG,MAASA,EAAK,CAAC,EAAG,OAAUA,EAAK,CAAC,CAAE,EAAG,OAAO,QAAW,aAAeD,EAAE,OAAO,QAAQ,EAAI,UAAW,CAAE,OAAO,IAAM,GAAIA,EACvJ,SAASC,EAAKC,EAAG,CAAE,OAAO,SAAUC,EAAG,CAAE,OAAOb,EAAK,CAACY,EAAGC,CAAC,CAAC,CAAG,CAAG,CACjE,SAASb,EAAKc,EAAI,CACd,GAAIN,EAAG,MAAM,IAAI,UAAU,iCAAiC,EAC5D,KAAOF,GAAG,GAAI,CACV,GAAIE,EAAI,EAAGC,IAAMF,EAAIO,EAAG,CAAC,EAAI,EAAIL,EAAE,OAAYK,EAAG,CAAC,EAAIL,EAAE,SAAcF,EAAIE,EAAE,SAAcF,EAAE,KAAKE,CAAC,EAAG,GAAKA,EAAE,OAAS,EAAEF,EAAIA,EAAE,KAAKE,EAAGK,EAAG,CAAC,CAAC,GAAG,KAAM,OAAOP,EAE3J,OADIE,EAAI,EAAGF,IAAGO,EAAK,CAACA,EAAG,CAAC,EAAI,EAAGP,EAAE,KAAK,GAC9BO,EAAG,CAAC,EAAG,CACX,IAAK,GAAG,IAAK,GAAGP,EAAIO,EAAI,MACxB,IAAK,GAAG,OAAAR,EAAE,QAAgB,CAAE,MAAOQ,EAAG,CAAC,EAAG,KAAM,EAAM,EACtD,IAAK,GAAGR,EAAE,QAASG,EAAIK,EAAG,CAAC,EAAGA,EAAK,CAAC,CAAC,EAAG,SACxC,IAAK,GAAGA,EAAKR,EAAE,IAAI,IAAI,EAAGA,EAAE,KAAK,IAAI,EAAG,SACxC,QACI,GAAMC,EAAID,EAAE,KAAM,EAAAC,EAAIA,EAAE,OAAS,GAAKA,EAAEA,EAAE,OAAS,CAAC,KAAOO,EAAG,CAAC,IAAM,GAAKA,EAAG,CAAC,IAAM,GAAI,CAAER,EAAI,EAAG,QAAU,CAC3G,GAAIQ,EAAG,CAAC,IAAM,IAAM,CAACP,GAAMO,EAAG,CAAC,EAAIP,EAAE,CAAC,GAAKO,EAAG,CAAC,EAAIP,EAAE,CAAC,GAAK,CAAED,EAAE,MAAQQ,EAAG,CAAC,EAAG,KAAO,CACrF,GAAIA,EAAG,CAAC,IAAM,GAAKR,EAAE,MAAQC,EAAE,CAAC,EAAG,CAAED,EAAE,MAAQC,EAAE,CAAC,EAAGA,EAAIO,EAAI,KAAO,CACpE,GAAIP,GAAKD,EAAE,MAAQC,EAAE,CAAC,EAAG,CAAED,EAAE,MAAQC,EAAE,CAAC,EAAGD,EAAE,IAAI,KAAKQ,CAAE,EAAG,KAAO,CAC9DP,EAAE,CAAC,GAAGD,EAAE,IAAI,IAAI,EACpBA,EAAE,KAAK,IAAI,EAAG,QACtB,CACAQ,EAAKT,EAAK,KAAKd,EAASe,CAAC,CAC7B,OAASL,EAAG,CAAEa,EAAK,CAAC,EAAGb,CAAC,EAAGQ,EAAI,CAAG,QAAE,CAAUD,EAAID,EAAI,CAAG,CACzD,GAAIO,EAAG,CAAC,EAAI,EAAG,MAAMA,EAAG,CAAC,EAAG,MAAO,CAAE,MAAOA,EAAG,CAAC,EAAIA,EAAG,CAAC,EAAI,OAAQ,KAAM,EAAK,CACnF,CACJ,CAcO,SAASC,GAASC,EAAG,CACxB,IAAIC,EAAI,OAAO,QAAW,YAAc,OAAO,SAAUC,EAAID,GAAKD,EAAEC,CAAC,EAAGE,EAAI,EAC5E,GAAID,EAAG,OAAOA,EAAE,KAAKF,CAAC,EACtB,GAAIA,GAAK,OAAOA,EAAE,QAAW,SAAU,MAAO,CAC1C,KAAM,UAAY,CACd,OAAIA,GAAKG,GAAKH,EAAE,SAAQA,EAAI,QACrB,CAAE,MAAOA,GAAKA,EAAEG,GAAG,EAAG,KAAM,CAACH,CAAE,CAC1C,CACJ,EACA,MAAM,IAAI,UAAUC,EAAI,0BAA4B,iCAAiC,CACzF,CAEO,SAASG,EAAOJ,EAAGK,EAAG,CACzB,IAAIH,EAAI,OAAO,QAAW,YAAcF,EAAE,OAAO,QAAQ,EACzD,GAAI,CAACE,EAAG,OAAOF,EACf,IAAIG,EAAID,EAAE,KAAKF,CAAC,EAAGM,EAAGC,EAAK,CAAC,EAAGC,EAC/B,GAAI,CACA,MAAQH,IAAM,QAAUA,KAAM,IAAM,EAAEC,EAAIH,EAAE,KAAK,GAAG,MAAMI,EAAG,KAAKD,EAAE,KAAK,CAC7E,OACOG,EAAO,CAAED,EAAI,CAAE,MAAOC,CAAM,CAAG,QACtC,CACI,GAAI,CACIH,GAAK,CAACA,EAAE,OAASJ,EAAIC,EAAE,SAAYD,EAAE,KAAKC,CAAC,CACnD,QACA,CAAU,GAAIK,EAAG,MAAMA,EAAE,KAAO,CACpC,CACA,OAAOD,CACX,CAkBO,SAASG,EAAcC,EAAIC,EAAMC,EAAM,CAC1C,GAAIA,GAAQ,UAAU,SAAW,EAAG,QAASC,EAAI,EAAGC,EAAIH,EAAK,OAAQI,EAAIF,EAAIC,EAAGD,KACxEE,GAAM,EAAEF,KAAKF,MACRI,IAAIA,EAAK,MAAM,UAAU,MAAM,KAAKJ,EAAM,EAAGE,CAAC,GACnDE,EAAGF,CAAC,EAAIF,EAAKE,CAAC,GAGtB,OAAOH,EAAG,OAAOK,GAAM,MAAM,UAAU,MAAM,KAAKJ,CAAI,CAAC,CAC3D,CAEO,SAASK,GAAQC,EAAG,CACvB,OAAO,gBAAgBD,IAAW,KAAK,EAAIC,EAAG,MAAQ,IAAID,GAAQC,CAAC,CACvE,CAEO,SAASC,GAAiBC,EAASC,EAAYC,EAAW,CAC7D,GAAI,CAAC,OAAO,cAAe,MAAM,IAAI,UAAU,sCAAsC,EACrF,IAAIC,EAAID,EAAU,MAAMF,EAASC,GAAc,CAAC,CAAC,EAAGP,EAAGU,EAAI,CAAC,EAC5D,OAAOV,EAAI,CAAC,EAAGW,EAAK,MAAM,EAAGA,EAAK,OAAO,EAAGA,EAAK,QAAQ,EAAGX,EAAE,OAAO,aAAa,EAAI,UAAY,CAAE,OAAO,IAAM,EAAGA,EACpH,SAASW,EAAKC,EAAG,CAAMH,EAAEG,CAAC,IAAGZ,EAAEY,CAAC,EAAI,SAAUR,EAAG,CAAE,OAAO,IAAI,QAAQ,SAAUS,EAAGC,EAAG,CAAEJ,EAAE,KAAK,CAACE,EAAGR,EAAGS,EAAGC,CAAC,CAAC,EAAI,GAAKC,EAAOH,EAAGR,CAAC,CAAG,CAAC,CAAG,EAAG,CACzI,SAASW,EAAOH,EAAGR,EAAG,CAAE,GAAI,CAAEY,EAAKP,EAAEG,CAAC,EAAER,CAAC,CAAC,CAAG,OAASa,EAAG,CAAEC,EAAOR,EAAE,CAAC,EAAE,CAAC,EAAGO,CAAC,CAAG,CAAE,CACjF,SAASD,EAAKG,EAAG,CAAEA,EAAE,iBAAiBhB,GAAU,QAAQ,QAAQgB,EAAE,MAAM,CAAC,EAAE,KAAKC,EAASC,CAAM,EAAIH,EAAOR,EAAE,CAAC,EAAE,CAAC,EAAGS,CAAC,CAAG,CACvH,SAASC,EAAQE,EAAO,CAAEP,EAAO,OAAQO,CAAK,CAAG,CACjD,SAASD,EAAOC,EAAO,CAAEP,EAAO,QAASO,CAAK,CAAG,CACjD,SAASJ,EAAOK,EAAGnB,EAAG,CAAMmB,EAAEnB,CAAC,EAAGM,EAAE,MAAM,EAAGA,EAAE,QAAQK,EAAOL,EAAE,CAAC,EAAE,CAAC,EAAGA,EAAE,CAAC,EAAE,CAAC,CAAC,CAAG,CACrF,CAQO,SAASc,GAAcC,EAAG,CAC7B,GAAI,CAAC,OAAO,cAAe,MAAM,IAAI,UAAU,sCAAsC,EACrF,IAAIC,EAAID,EAAE,OAAO,aAAa,EAAGE,EACjC,OAAOD,EAAIA,EAAE,KAAKD,CAAC,GAAKA,EAAI,OAAOG,IAAa,WAAaA,GAASH,CAAC,EAAIA,EAAE,OAAO,QAAQ,EAAE,EAAGE,EAAI,CAAC,EAAGE,EAAK,MAAM,EAAGA,EAAK,OAAO,EAAGA,EAAK,QAAQ,EAAGF,EAAE,OAAO,aAAa,EAAI,UAAY,CAAE,OAAO,IAAM,EAAGA,GAC9M,SAASE,EAAKC,EAAG,CAAEH,EAAEG,CAAC,EAAIL,EAAEK,CAAC,GAAK,SAAUC,EAAG,CAAE,OAAO,IAAI,QAAQ,SAAUC,EAASC,EAAQ,CAAEF,EAAIN,EAAEK,CAAC,EAAEC,CAAC,EAAGG,EAAOF,EAASC,EAAQF,EAAE,KAAMA,EAAE,KAAK,CAAG,CAAC,CAAG,CAAG,CAC/J,SAASG,EAAOF,EAASC,EAAQE,EAAGJ,EAAG,CAAE,QAAQ,QAAQA,CAAC,EAAE,KAAK,SAASA,EAAG,CAAEC,EAAQ,CAAE,MAAOD,EAAG,KAAMI,CAAE,CAAC,CAAG,EAAGF,CAAM,CAAG,CAC/H,CCtMM,SAAUG,EAAWC,EAAU,CACnC,OAAO,OAAOA,GAAU,UAC1B,CCGM,SAAUC,GAAoBC,EAAgC,CAClE,IAAMC,EAAS,SAACC,EAAa,CAC3B,MAAM,KAAKA,CAAQ,EACnBA,EAAS,MAAQ,IAAI,MAAK,EAAG,KAC/B,EAEMC,EAAWH,EAAWC,CAAM,EAClC,OAAAE,EAAS,UAAY,OAAO,OAAO,MAAM,SAAS,EAClDA,EAAS,UAAU,YAAcA,EAC1BA,CACT,CCDO,IAAMC,GAA+CC,GAC1D,SAACC,EAAM,CACL,OAAA,SAA4CC,EAA0B,CACpED,EAAO,IAAI,EACX,KAAK,QAAUC,EACRA,EAAO,OAAM;EACxBA,EAAO,IAAI,SAACC,EAAKC,EAAC,CAAK,OAAGA,EAAI,EAAC,KAAKD,EAAI,SAAQ,CAAzB,CAA6B,EAAE,KAAK;GAAM,EACzD,GACJ,KAAK,KAAO,sBACZ,KAAK,OAASD,CAChB,CARA,CAQC,ECvBC,SAAUG,GAAaC,EAA6BC,EAAO,CAC/D,GAAID,EAAK,CACP,IAAME,EAAQF,EAAI,QAAQC,CAAI,EAC9B,GAAKC,GAASF,EAAI,OAAOE,EAAO,CAAC,EAErC,CCOA,IAAAC,GAAA,UAAA,CAyBE,SAAAA,EAAoBC,EAA4B,CAA5B,KAAA,gBAAAA,EAdb,KAAA,OAAS,GAER,KAAA,WAAmD,KAMnD,KAAA,YAAqD,IAMV,CAQnD,OAAAD,EAAA,UAAA,YAAA,UAAA,aACME,EAEJ,GAAI,CAAC,KAAK,OAAQ,CAChB,KAAK,OAAS,GAGN,IAAAC,EAAe,KAAI,WAC3B,GAAIA,EAEF,GADA,KAAK,WAAa,KACd,MAAM,QAAQA,CAAU,MAC1B,QAAqBC,EAAAC,GAAAF,CAAU,EAAAG,EAAAF,EAAA,KAAA,EAAA,CAAAE,EAAA,KAAAA,EAAAF,EAAA,KAAA,EAAE,CAA5B,IAAMG,EAAMD,EAAA,MACfC,EAAO,OAAO,IAAI,yGAGpBJ,EAAW,OAAO,IAAI,EAIlB,IAAiBK,EAAqB,KAAI,gBAClD,GAAIC,EAAWD,CAAgB,EAC7B,GAAI,CACFA,EAAgB,QACTE,EAAG,CACVR,EAASQ,aAAaC,GAAsBD,EAAE,OAAS,CAACA,CAAC,EAIrD,IAAAE,EAAgB,KAAI,YAC5B,GAAIA,EAAa,CACf,KAAK,YAAc,SACnB,QAAwBC,EAAAR,GAAAO,CAAW,EAAAE,EAAAD,EAAA,KAAA,EAAA,CAAAC,EAAA,KAAAA,EAAAD,EAAA,KAAA,EAAE,CAAhC,IAAME,EAASD,EAAA,MAClB,GAAI,CACFE,GAAcD,CAAS,QAChBE,EAAK,CACZf,EAASA,GAAM,KAANA,EAAU,CAAA,EACfe,aAAeN,GACjBT,EAAMgB,EAAAA,EAAA,CAAA,EAAAC,EAAOjB,CAAM,CAAA,EAAAiB,EAAKF,EAAI,MAAM,CAAA,EAElCf,EAAO,KAAKe,CAAG,sGAMvB,GAAIf,EACF,MAAM,IAAIS,GAAoBT,CAAM,EAG1C,EAoBAF,EAAA,UAAA,IAAA,SAAIoB,EAAuB,OAGzB,GAAIA,GAAYA,IAAa,KAC3B,GAAI,KAAK,OAGPJ,GAAcI,CAAQ,MACjB,CACL,GAAIA,aAAoBpB,EAAc,CAGpC,GAAIoB,EAAS,QAAUA,EAAS,WAAW,IAAI,EAC7C,OAEFA,EAAS,WAAW,IAAI,GAEzB,KAAK,aAAcC,EAAA,KAAK,eAAW,MAAAA,IAAA,OAAAA,EAAI,CAAA,GAAI,KAAKD,CAAQ,EAG/D,EAOQpB,EAAA,UAAA,WAAR,SAAmBsB,EAAoB,CAC7B,IAAAnB,EAAe,KAAI,WAC3B,OAAOA,IAAemB,GAAW,MAAM,QAAQnB,CAAU,GAAKA,EAAW,SAASmB,CAAM,CAC1F,EASQtB,EAAA,UAAA,WAAR,SAAmBsB,EAAoB,CAC7B,IAAAnB,EAAe,KAAI,WAC3B,KAAK,WAAa,MAAM,QAAQA,CAAU,GAAKA,EAAW,KAAKmB,CAAM,EAAGnB,GAAcA,EAAa,CAACA,EAAYmB,CAAM,EAAIA,CAC5H,EAMQtB,EAAA,UAAA,cAAR,SAAsBsB,EAAoB,CAChC,IAAAnB,EAAe,KAAI,WACvBA,IAAemB,EACjB,KAAK,WAAa,KACT,MAAM,QAAQnB,CAAU,GACjCoB,GAAUpB,EAAYmB,CAAM,CAEhC,EAgBAtB,EAAA,UAAA,OAAA,SAAOoB,EAAsC,CACnC,IAAAR,EAAgB,KAAI,YAC5BA,GAAeW,GAAUX,EAAaQ,CAAQ,EAE1CA,aAAoBpB,GACtBoB,EAAS,cAAc,IAAI,CAE/B,EAlLcpB,EAAA,MAAS,UAAA,CACrB,IAAMwB,EAAQ,IAAIxB,EAClB,OAAAwB,EAAM,OAAS,GACRA,CACT,EAAE,EA+KJxB,GArLA,EAuLO,IAAMyB,GAAqBC,GAAa,MAEzC,SAAUC,GAAeC,EAAU,CACvC,OACEA,aAAiBF,IAChBE,GAAS,WAAYA,GAASC,EAAWD,EAAM,MAAM,GAAKC,EAAWD,EAAM,GAAG,GAAKC,EAAWD,EAAM,WAAW,CAEpH,CAEA,SAASE,GAAcC,EAAwC,CACzDF,EAAWE,CAAS,EACtBA,EAAS,EAETA,EAAU,YAAW,CAEzB,CChNO,IAAMC,GAAuB,CAClC,iBAAkB,KAClB,sBAAuB,KACvB,QAAS,OACT,sCAAuC,GACvC,yBAA0B,ICGrB,IAAMC,GAAmC,CAG9C,WAAA,SAAWC,EAAqBC,EAAgB,SAAEC,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,EAAA,CAAA,EAAA,UAAAA,CAAA,EACxC,IAAAC,EAAaL,GAAe,SACpC,OAAIK,GAAQ,MAARA,EAAU,WACLA,EAAS,WAAU,MAAnBA,EAAQC,EAAA,CAAYL,EAASC,CAAO,EAAAK,EAAKJ,CAAI,CAAA,CAAA,EAE/C,WAAU,MAAA,OAAAG,EAAA,CAACL,EAASC,CAAO,EAAAK,EAAKJ,CAAI,CAAA,CAAA,CAC7C,EACA,aAAA,SAAaK,EAAM,CACT,IAAAH,EAAaL,GAAe,SACpC,QAAQK,GAAQ,KAAA,OAARA,EAAU,eAAgB,cAAcG,CAAa,CAC/D,EACA,SAAU,QCjBN,SAAUC,GAAqBC,EAAQ,CAC3CC,GAAgB,WAAW,UAAA,CACjB,IAAAC,EAAqBC,GAAM,iBACnC,GAAID,EAEFA,EAAiBF,CAAG,MAGpB,OAAMA,CAEV,CAAC,CACH,CCtBM,SAAUI,IAAI,CAAK,CCMlB,IAAMC,GAAyB,UAAA,CAAM,OAAAC,GAAmB,IAAK,OAAW,MAAS,CAA5C,EAAsE,EAO5G,SAAUC,GAAkBC,EAAU,CAC1C,OAAOF,GAAmB,IAAK,OAAWE,CAAK,CACjD,CAOM,SAAUC,GAAoBC,EAAQ,CAC1C,OAAOJ,GAAmB,IAAKI,EAAO,MAAS,CACjD,CAQM,SAAUJ,GAAmBK,EAAuBD,EAAYF,EAAU,CAC9E,MAAO,CACL,KAAIG,EACJ,MAAKD,EACL,MAAKF,EAET,CCrCA,IAAII,GAAuD,KASrD,SAAUC,GAAaC,EAAc,CACzC,GAAIC,GAAO,sCAAuC,CAChD,IAAMC,EAAS,CAACJ,GAKhB,GAJII,IACFJ,GAAU,CAAE,YAAa,GAAO,MAAO,IAAI,GAE7CE,EAAE,EACEE,EAAQ,CACJ,IAAAC,EAAyBL,GAAvBM,EAAWD,EAAA,YAAEE,EAAKF,EAAA,MAE1B,GADAL,GAAU,KACNM,EACF,MAAMC,QAMVL,EAAE,CAEN,CAMM,SAAUM,GAAaC,EAAQ,CAC/BN,GAAO,uCAAyCH,KAClDA,GAAQ,YAAc,GACtBA,GAAQ,MAAQS,EAEpB,CCrBA,IAAAC,GAAA,SAAAC,EAAA,CAAmCC,GAAAF,EAAAC,CAAA,EA6BjC,SAAAD,EAAYG,EAA6C,CAAzD,IAAAC,EACEH,EAAA,KAAA,IAAA,GAAO,KATC,OAAAG,EAAA,UAAqB,GAUzBD,GACFC,EAAK,YAAcD,EAGfE,GAAeF,CAAW,GAC5BA,EAAY,IAAIC,CAAI,GAGtBA,EAAK,YAAcE,IAEvB,CAzBO,OAAAN,EAAA,OAAP,SAAiBO,EAAwBC,EAA2BC,EAAqB,CACvF,OAAO,IAAIC,GAAeH,EAAMC,EAAOC,CAAQ,CACjD,EAgCAT,EAAA,UAAA,KAAA,SAAKW,EAAS,CACR,KAAK,UACPC,GAA0BC,GAAiBF,CAAK,EAAG,IAAI,EAEvD,KAAK,MAAMA,CAAM,CAErB,EASAX,EAAA,UAAA,MAAA,SAAMc,EAAS,CACT,KAAK,UACPF,GAA0BG,GAAkBD,CAAG,EAAG,IAAI,GAEtD,KAAK,UAAY,GACjB,KAAK,OAAOA,CAAG,EAEnB,EAQAd,EAAA,UAAA,SAAA,UAAA,CACM,KAAK,UACPY,GAA0BI,GAAuB,IAAI,GAErD,KAAK,UAAY,GACjB,KAAK,UAAS,EAElB,EAEAhB,EAAA,UAAA,YAAA,UAAA,CACO,KAAK,SACR,KAAK,UAAY,GACjBC,EAAA,UAAM,YAAW,KAAA,IAAA,EACjB,KAAK,YAAc,KAEvB,EAEUD,EAAA,UAAA,MAAV,SAAgBW,EAAQ,CACtB,KAAK,YAAY,KAAKA,CAAK,CAC7B,EAEUX,EAAA,UAAA,OAAV,SAAiBc,EAAQ,CACvB,GAAI,CACF,KAAK,YAAY,MAAMA,CAAG,UAE1B,KAAK,YAAW,EAEpB,EAEUd,EAAA,UAAA,UAAV,UAAA,CACE,GAAI,CACF,KAAK,YAAY,SAAQ,UAEzB,KAAK,YAAW,EAEpB,EACFA,CAAA,EApHmCiB,EAAY,EA2H/C,IAAMC,GAAQ,SAAS,UAAU,KAEjC,SAASC,GAAyCC,EAAQC,EAAY,CACpE,OAAOH,GAAM,KAAKE,EAAIC,CAAO,CAC/B,CAMA,IAAAC,GAAA,UAAA,CACE,SAAAA,EAAoBC,EAAqC,CAArC,KAAA,gBAAAA,CAAwC,CAE5D,OAAAD,EAAA,UAAA,KAAA,SAAKE,EAAQ,CACH,IAAAD,EAAoB,KAAI,gBAChC,GAAIA,EAAgB,KAClB,GAAI,CACFA,EAAgB,KAAKC,CAAK,QACnBC,EAAO,CACdC,GAAqBD,CAAK,EAGhC,EAEAH,EAAA,UAAA,MAAA,SAAMK,EAAQ,CACJ,IAAAJ,EAAoB,KAAI,gBAChC,GAAIA,EAAgB,MAClB,GAAI,CACFA,EAAgB,MAAMI,CAAG,QAClBF,EAAO,CACdC,GAAqBD,CAAK,OAG5BC,GAAqBC,CAAG,CAE5B,EAEAL,EAAA,UAAA,SAAA,UAAA,CACU,IAAAC,EAAoB,KAAI,gBAChC,GAAIA,EAAgB,SAClB,GAAI,CACFA,EAAgB,SAAQ,QACjBE,EAAO,CACdC,GAAqBD,CAAK,EAGhC,EACFH,CAAA,EArCA,EAuCAM,GAAA,SAAAC,EAAA,CAAuCC,GAAAF,EAAAC,CAAA,EACrC,SAAAD,EACEG,EACAN,EACAO,EAA8B,CAHhC,IAAAC,EAKEJ,EAAA,KAAA,IAAA,GAAO,KAEHN,EACJ,GAAIW,EAAWH,CAAc,GAAK,CAACA,EAGjCR,EAAkB,CAChB,KAAOQ,GAAc,KAAdA,EAAkB,OACzB,MAAON,GAAK,KAALA,EAAS,OAChB,SAAUO,GAAQ,KAARA,EAAY,YAEnB,CAEL,IAAIG,EACAF,GAAQG,GAAO,0BAIjBD,EAAU,OAAO,OAAOJ,CAAc,EACtCI,EAAQ,YAAc,UAAA,CAAM,OAAAF,EAAK,YAAW,CAAhB,EAC5BV,EAAkB,CAChB,KAAMQ,EAAe,MAAQZ,GAAKY,EAAe,KAAMI,CAAO,EAC9D,MAAOJ,EAAe,OAASZ,GAAKY,EAAe,MAAOI,CAAO,EACjE,SAAUJ,EAAe,UAAYZ,GAAKY,EAAe,SAAUI,CAAO,IAI5EZ,EAAkBQ,EAMtB,OAAAE,EAAK,YAAc,IAAIX,GAAiBC,CAAe,GACzD,CACF,OAAAK,CAAA,EAzCuCS,EAAU,EA2CjD,SAASC,GAAqBC,EAAU,CAClCC,GAAO,sCACTC,GAAaF,CAAK,EAIlBG,GAAqBH,CAAK,CAE9B,CAQA,SAASI,GAAoBC,EAAQ,CACnC,MAAMA,CACR,CAOA,SAASC,GAA0BC,EAA2CC,EAA2B,CAC/F,IAAAC,EAA0BR,GAAM,sBACxCQ,GAAyBC,GAAgB,WAAW,UAAA,CAAM,OAAAD,EAAsBF,EAAcC,CAAU,CAA9C,CAA+C,CAC3G,CAOO,IAAMG,GAA6D,CACxE,OAAQ,GACR,KAAMC,GACN,MAAOR,GACP,SAAUQ,IC5QL,IAAMC,GAA+B,UAAA,CAAM,OAAC,OAAO,QAAW,YAAc,OAAO,YAAe,cAAvD,EAAsE,ECoClH,SAAUC,GAAYC,EAAI,CAC9B,OAAOA,CACT,CCiCM,SAAUC,IAAI,SAACC,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,CAAA,EAAA,UAAAA,CAAA,EACnB,OAAOC,GAAcF,CAAG,CAC1B,CAGM,SAAUE,GAAoBF,EAA+B,CACjE,OAAIA,EAAI,SAAW,EACVG,GAGLH,EAAI,SAAW,EACVA,EAAI,CAAC,EAGP,SAAeI,EAAQ,CAC5B,OAAOJ,EAAI,OAAO,SAACK,EAAWC,EAAuB,CAAK,OAAAA,EAAGD,CAAI,CAAP,EAAUD,CAAY,CAClF,CACF,CC9EA,IAAAG,EAAA,UAAA,CAkBE,SAAAA,EAAYC,EAA6E,CACnFA,IACF,KAAK,WAAaA,EAEtB,CA4BA,OAAAD,EAAA,UAAA,KAAA,SAAQE,EAAyB,CAC/B,IAAMC,EAAa,IAAIH,EACvB,OAAAG,EAAW,OAAS,KACpBA,EAAW,SAAWD,EACfC,CACT,EA6IAH,EAAA,UAAA,UAAA,SACEI,EACAC,EACAC,EAA8B,CAHhC,IAAAC,EAAA,KAKQC,EAAaC,GAAaL,CAAc,EAAIA,EAAiB,IAAIM,GAAeN,EAAgBC,EAAOC,CAAQ,EAErH,OAAAK,GAAa,UAAA,CACL,IAAAC,EAAuBL,EAArBL,EAAQU,EAAA,SAAEC,EAAMD,EAAA,OACxBJ,EAAW,IACTN,EAGIA,EAAS,KAAKM,EAAYK,CAAM,EAChCA,EAIAN,EAAK,WAAWC,CAAU,EAG1BD,EAAK,cAAcC,CAAU,CAAC,CAEtC,CAAC,EAEMA,CACT,EAGUR,EAAA,UAAA,cAAV,SAAwBc,EAAmB,CACzC,GAAI,CACF,OAAO,KAAK,WAAWA,CAAI,QACpBC,EAAK,CAIZD,EAAK,MAAMC,CAAG,EAElB,EA6DAf,EAAA,UAAA,QAAA,SAAQgB,EAA0BC,EAAoC,CAAtE,IAAAV,EAAA,KACE,OAAAU,EAAcC,GAAeD,CAAW,EAEjC,IAAIA,EAAkB,SAACE,EAASC,EAAM,CAC3C,IAAMZ,EAAa,IAAIE,GAAkB,CACvC,KAAM,SAACW,EAAK,CACV,GAAI,CACFL,EAAKK,CAAK,QACHN,EAAK,CACZK,EAAOL,CAAG,EACVP,EAAW,YAAW,EAE1B,EACA,MAAOY,EACP,SAAUD,EACX,EACDZ,EAAK,UAAUC,CAAU,CAC3B,CAAC,CACH,EAGUR,EAAA,UAAA,WAAV,SAAqBQ,EAA2B,OAC9C,OAAOI,EAAA,KAAK,UAAM,MAAAA,IAAA,OAAA,OAAAA,EAAE,UAAUJ,CAAU,CAC1C,EAOAR,EAAA,UAACG,EAAiB,EAAlB,UAAA,CACE,OAAO,IACT,EA4FAH,EAAA,UAAA,KAAA,UAAA,SAAKsB,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,CAAA,EAAA,UAAAA,CAAA,EACH,OAAOC,GAAcF,CAAU,EAAE,IAAI,CACvC,EA6BAtB,EAAA,UAAA,UAAA,SAAUiB,EAAoC,CAA9C,IAAAV,EAAA,KACE,OAAAU,EAAcC,GAAeD,CAAW,EAEjC,IAAIA,EAAY,SAACE,EAASC,EAAM,CACrC,IAAIC,EACJd,EAAK,UACH,SAACkB,EAAI,CAAK,OAACJ,EAAQI,CAAT,EACV,SAACV,EAAQ,CAAK,OAAAK,EAAOL,CAAG,CAAV,EACd,UAAA,CAAM,OAAAI,EAAQE,CAAK,CAAb,CAAc,CAExB,CAAC,CACH,EA1aOrB,EAAA,OAAkC,SAAIC,EAAwD,CACnG,OAAO,IAAID,EAAcC,CAAS,CACpC,EAyaFD,GA9cA,EAudA,SAAS0B,GAAeC,EAA+C,OACrE,OAAOC,EAAAD,GAAW,KAAXA,EAAeE,GAAO,WAAO,MAAAD,IAAA,OAAAA,EAAI,OAC1C,CAEA,SAASE,GAAcC,EAAU,CAC/B,OAAOA,GAASC,EAAWD,EAAM,IAAI,GAAKC,EAAWD,EAAM,KAAK,GAAKC,EAAWD,EAAM,QAAQ,CAChG,CAEA,SAASE,GAAgBF,EAAU,CACjC,OAAQA,GAASA,aAAiBG,IAAgBJ,GAAWC,CAAK,GAAKI,GAAeJ,CAAK,CAC7F,CCzeM,SAAUK,GAAQC,EAAW,CACjC,OAAOC,EAAWD,GAAM,KAAA,OAANA,EAAQ,IAAI,CAChC,CAMM,SAAUE,EACdC,EAAqF,CAErF,OAAO,SAACH,EAAqB,CAC3B,GAAID,GAAQC,CAAM,EAChB,OAAOA,EAAO,KAAK,SAA+BI,EAA2B,CAC3E,GAAI,CACF,OAAOD,EAAKC,EAAc,IAAI,QACvBC,EAAK,CACZ,KAAK,MAAMA,CAAG,EAElB,CAAC,EAEH,MAAM,IAAI,UAAU,wCAAwC,CAC9D,CACF,CCjBM,SAAUC,EACdC,EACAC,EACAC,EACAC,EACAC,EAAuB,CAEvB,OAAO,IAAIC,GAAmBL,EAAaC,EAAQC,EAAYC,EAASC,CAAU,CACpF,CAMA,IAAAC,GAAA,SAAAC,EAAA,CAA2CC,GAAAF,EAAAC,CAAA,EAiBzC,SAAAD,EACEL,EACAC,EACAC,EACAC,EACQC,EACAI,EAAiC,CAN3C,IAAAC,EAoBEH,EAAA,KAAA,KAAMN,CAAW,GAAC,KAfV,OAAAS,EAAA,WAAAL,EACAK,EAAA,kBAAAD,EAeRC,EAAK,MAAQR,EACT,SAAuCS,EAAQ,CAC7C,GAAI,CACFT,EAAOS,CAAK,QACLC,EAAK,CACZX,EAAY,MAAMW,CAAG,EAEzB,EACAL,EAAA,UAAM,MACVG,EAAK,OAASN,EACV,SAAuCQ,EAAQ,CAC7C,GAAI,CACFR,EAAQQ,CAAG,QACJA,EAAK,CAEZX,EAAY,MAAMW,CAAG,UAGrB,KAAK,YAAW,EAEpB,EACAL,EAAA,UAAM,OACVG,EAAK,UAAYP,EACb,UAAA,CACE,GAAI,CACFA,EAAU,QACHS,EAAK,CAEZX,EAAY,MAAMW,CAAG,UAGrB,KAAK,YAAW,EAEpB,EACAL,EAAA,UAAM,WACZ,CAEA,OAAAD,EAAA,UAAA,YAAA,UAAA,OACE,GAAI,CAAC,KAAK,mBAAqB,KAAK,kBAAiB,EAAI,CAC/C,IAAAO,EAAW,KAAI,OACvBN,EAAA,UAAM,YAAW,KAAA,IAAA,EAEjB,CAACM,KAAUC,EAAA,KAAK,cAAU,MAAAA,IAAA,QAAAA,EAAA,KAAf,IAAI,GAEnB,EACFR,CAAA,EAnF2CS,EAAU,ECd9C,IAAMC,GAAiD,CAG5D,SAAA,SAASC,EAAQ,CACf,IAAIC,EAAU,sBACVC,EAAkD,qBAC9CC,EAAaJ,GAAsB,SACvCI,IACFF,EAAUE,EAAS,sBACnBD,EAASC,EAAS,sBAEpB,IAAMC,EAASH,EAAQ,SAACI,EAAS,CAI/BH,EAAS,OACTF,EAASK,CAAS,CACpB,CAAC,EACD,OAAO,IAAIC,GAAa,UAAA,CAAM,OAAAJ,GAAM,KAAA,OAANA,EAASE,CAAM,CAAf,CAAgB,CAChD,EACA,sBAAqB,UAAA,SAACG,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,CAAA,EAAA,UAAAA,CAAA,EACZ,IAAAL,EAAaJ,GAAsB,SAC3C,QAAQI,GAAQ,KAAA,OAARA,EAAU,wBAAyB,uBAAsB,MAAA,OAAAM,EAAA,CAAA,EAAAC,EAAIH,CAAI,CAAA,CAAA,CAC3E,EACA,qBAAoB,UAAA,SAACA,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,CAAA,EAAA,UAAAA,CAAA,EACX,IAAAL,EAAaJ,GAAsB,SAC3C,QAAQI,GAAQ,KAAA,OAARA,EAAU,uBAAwB,sBAAqB,MAAA,OAAAM,EAAA,CAAA,EAAAC,EAAIH,CAAI,CAAA,CAAA,CACzE,EACA,SAAU,QCrBL,IAAMI,GAAuDC,GAClE,SAACC,EAAM,CACL,OAAA,UAAoC,CAClCA,EAAO,IAAI,EACX,KAAK,KAAO,0BACZ,KAAK,QAAU,qBACjB,CAJA,CAIC,ECXL,IAAAC,EAAA,SAAAC,EAAA,CAAgCC,GAAAF,EAAAC,CAAA,EAwB9B,SAAAD,GAAA,CAAA,IAAAG,EAEEF,EAAA,KAAA,IAAA,GAAO,KAzBT,OAAAE,EAAA,OAAS,GAEDA,EAAA,iBAAyC,KAGjDA,EAAA,UAA2B,CAAA,EAE3BA,EAAA,UAAY,GAEZA,EAAA,SAAW,GAEXA,EAAA,YAAmB,MAenB,CAGA,OAAAH,EAAA,UAAA,KAAA,SAAQI,EAAwB,CAC9B,IAAMC,EAAU,IAAIC,GAAiB,KAAM,IAAI,EAC/C,OAAAD,EAAQ,SAAWD,EACZC,CACT,EAGUL,EAAA,UAAA,eAAV,UAAA,CACE,GAAI,KAAK,OACP,MAAM,IAAIO,EAEd,EAEAP,EAAA,UAAA,KAAA,SAAKQ,EAAQ,CAAb,IAAAL,EAAA,KACEM,GAAa,UAAA,SAEX,GADAN,EAAK,eAAc,EACf,CAACA,EAAK,UAAW,CACdA,EAAK,mBACRA,EAAK,iBAAmB,MAAM,KAAKA,EAAK,SAAS,OAEnD,QAAuBO,EAAAC,GAAAR,EAAK,gBAAgB,EAAAS,EAAAF,EAAA,KAAA,EAAA,CAAAE,EAAA,KAAAA,EAAAF,EAAA,KAAA,EAAE,CAAzC,IAAMG,EAAQD,EAAA,MACjBC,EAAS,KAAKL,CAAK,qGAGzB,CAAC,CACH,EAEAR,EAAA,UAAA,MAAA,SAAMc,EAAQ,CAAd,IAAAX,EAAA,KACEM,GAAa,UAAA,CAEX,GADAN,EAAK,eAAc,EACf,CAACA,EAAK,UAAW,CACnBA,EAAK,SAAWA,EAAK,UAAY,GACjCA,EAAK,YAAcW,EAEnB,QADQC,EAAcZ,EAAI,UACnBY,EAAU,QACfA,EAAU,MAAK,EAAI,MAAMD,CAAG,EAGlC,CAAC,CACH,EAEAd,EAAA,UAAA,SAAA,UAAA,CAAA,IAAAG,EAAA,KACEM,GAAa,UAAA,CAEX,GADAN,EAAK,eAAc,EACf,CAACA,EAAK,UAAW,CACnBA,EAAK,UAAY,GAEjB,QADQY,EAAcZ,EAAI,UACnBY,EAAU,QACfA,EAAU,MAAK,EAAI,SAAQ,EAGjC,CAAC,CACH,EAEAf,EAAA,UAAA,YAAA,UAAA,CACE,KAAK,UAAY,KAAK,OAAS,GAC/B,KAAK,UAAY,KAAK,iBAAmB,IAC3C,EAEA,OAAA,eAAIA,EAAA,UAAA,WAAQ,KAAZ,UAAA,OACE,QAAOgB,EAAA,KAAK,aAAS,MAAAA,IAAA,OAAA,OAAAA,EAAE,QAAS,CAClC,kCAGUhB,EAAA,UAAA,cAAV,SAAwBiB,EAAyB,CAC/C,YAAK,eAAc,EACZhB,EAAA,UAAM,cAAa,KAAA,KAACgB,CAAU,CACvC,EAGUjB,EAAA,UAAA,WAAV,SAAqBiB,EAAyB,CAC5C,YAAK,eAAc,EACnB,KAAK,wBAAwBA,CAAU,EAChC,KAAK,gBAAgBA,CAAU,CACxC,EAGUjB,EAAA,UAAA,gBAAV,SAA0BiB,EAA2B,CAArD,IAAAd,EAAA,KACQa,EAAqC,KAAnCE,EAAQF,EAAA,SAAEG,EAASH,EAAA,UAAED,EAASC,EAAA,UACtC,OAAIE,GAAYC,EACPC,IAET,KAAK,iBAAmB,KACxBL,EAAU,KAAKE,CAAU,EAClB,IAAII,GAAa,UAAA,CACtBlB,EAAK,iBAAmB,KACxBmB,GAAUP,EAAWE,CAAU,CACjC,CAAC,EACH,EAGUjB,EAAA,UAAA,wBAAV,SAAkCiB,EAA2B,CACrD,IAAAD,EAAuC,KAArCE,EAAQF,EAAA,SAAEO,EAAWP,EAAA,YAAEG,EAASH,EAAA,UACpCE,EACFD,EAAW,MAAMM,CAAW,EACnBJ,GACTF,EAAW,SAAQ,CAEvB,EAQAjB,EAAA,UAAA,aAAA,UAAA,CACE,IAAMwB,EAAkB,IAAIC,EAC5B,OAAAD,EAAW,OAAS,KACbA,CACT,EAxHOxB,EAAA,OAAkC,SAAI0B,EAA0BC,EAAqB,CAC1F,OAAO,IAAIrB,GAAoBoB,EAAaC,CAAM,CACpD,EAuHF3B,GA7IgCyB,CAAU,EAkJ1C,IAAAG,GAAA,SAAAC,EAAA,CAAyCC,GAAAF,EAAAC,CAAA,EACvC,SAAAD,EAESG,EACPC,EAAsB,CAHxB,IAAAC,EAKEJ,EAAA,KAAA,IAAA,GAAO,KAHA,OAAAI,EAAA,YAAAF,EAIPE,EAAK,OAASD,GAChB,CAEA,OAAAJ,EAAA,UAAA,KAAA,SAAKM,EAAQ,UACXC,GAAAC,EAAA,KAAK,eAAW,MAAAA,IAAA,OAAA,OAAAA,EAAE,QAAI,MAAAD,IAAA,QAAAA,EAAA,KAAAC,EAAGF,CAAK,CAChC,EAEAN,EAAA,UAAA,MAAA,SAAMS,EAAQ,UACZF,GAAAC,EAAA,KAAK,eAAW,MAAAA,IAAA,OAAA,OAAAA,EAAE,SAAK,MAAAD,IAAA,QAAAA,EAAA,KAAAC,EAAGC,CAAG,CAC/B,EAEAT,EAAA,UAAA,SAAA,UAAA,UACEO,GAAAC,EAAA,KAAK,eAAW,MAAAA,IAAA,OAAA,OAAAA,EAAE,YAAQ,MAAAD,IAAA,QAAAA,EAAA,KAAAC,CAAA,CAC5B,EAGUR,EAAA,UAAA,WAAV,SAAqBU,EAAyB,SAC5C,OAAOH,GAAAC,EAAA,KAAK,UAAM,MAAAA,IAAA,OAAA,OAAAA,EAAE,UAAUE,CAAU,KAAC,MAAAH,IAAA,OAAAA,EAAII,EAC/C,EACFX,CAAA,EA1ByCY,CAAO,EC5JzC,IAAMC,GAA+C,CAC1D,IAAG,UAAA,CAGD,OAAQA,GAAsB,UAAY,MAAM,IAAG,CACrD,EACA,SAAU,QCwBZ,IAAAC,GAAA,SAAAC,EAAA,CAAsCC,GAAAF,EAAAC,CAAA,EAUpC,SAAAD,EACUG,EACAC,EACAC,EAA6D,CAF7DF,IAAA,SAAAA,EAAA,KACAC,IAAA,SAAAA,EAAA,KACAC,IAAA,SAAAA,EAAAC,IAHV,IAAAC,EAKEN,EAAA,KAAA,IAAA,GAAO,KAJC,OAAAM,EAAA,YAAAJ,EACAI,EAAA,YAAAH,EACAG,EAAA,mBAAAF,EAZFE,EAAA,QAA0B,CAAA,EAC1BA,EAAA,oBAAsB,GAc5BA,EAAK,oBAAsBH,IAAgB,IAC3CG,EAAK,YAAc,KAAK,IAAI,EAAGJ,CAAW,EAC1CI,EAAK,YAAc,KAAK,IAAI,EAAGH,CAAW,GAC5C,CAEA,OAAAJ,EAAA,UAAA,KAAA,SAAKQ,EAAQ,CACL,IAAAC,EAA+E,KAA7EC,EAASD,EAAA,UAAEE,EAAOF,EAAA,QAAEG,EAAmBH,EAAA,oBAAEJ,EAAkBI,EAAA,mBAAEL,EAAWK,EAAA,YAC3EC,IACHC,EAAQ,KAAKH,CAAK,EAClB,CAACI,GAAuBD,EAAQ,KAAKN,EAAmB,IAAG,EAAKD,CAAW,GAE7E,KAAK,YAAW,EAChBH,EAAA,UAAM,KAAI,KAAA,KAACO,CAAK,CAClB,EAGUR,EAAA,UAAA,WAAV,SAAqBa,EAAyB,CAC5C,KAAK,eAAc,EACnB,KAAK,YAAW,EAQhB,QANMC,EAAe,KAAK,gBAAgBD,CAAU,EAE9CJ,EAAmC,KAAjCG,EAAmBH,EAAA,oBAAEE,EAAOF,EAAA,QAG9BM,EAAOJ,EAAQ,MAAK,EACjBK,EAAI,EAAGA,EAAID,EAAK,QAAU,CAACF,EAAW,OAAQG,GAAKJ,EAAsB,EAAI,EACpFC,EAAW,KAAKE,EAAKC,CAAC,CAAM,EAG9B,YAAK,wBAAwBH,CAAU,EAEhCC,CACT,EAEQd,EAAA,UAAA,YAAR,UAAA,CACQ,IAAAS,EAAoE,KAAlEN,EAAWM,EAAA,YAAEJ,EAAkBI,EAAA,mBAAEE,EAAOF,EAAA,QAAEG,EAAmBH,EAAA,oBAK/DQ,GAAsBL,EAAsB,EAAI,GAAKT,EAK3D,GAJAA,EAAc,KAAYc,EAAqBN,EAAQ,QAAUA,EAAQ,OAAO,EAAGA,EAAQ,OAASM,CAAkB,EAIlH,CAACL,EAAqB,CAKxB,QAJMM,EAAMb,EAAmB,IAAG,EAC9Bc,EAAO,EAGFH,EAAI,EAAGA,EAAIL,EAAQ,QAAWA,EAAQK,CAAC,GAAgBE,EAAKF,GAAK,EACxEG,EAAOH,EAETG,GAAQR,EAAQ,OAAO,EAAGQ,EAAO,CAAC,EAEtC,EACFnB,CAAA,EAzEsCoB,CAAO,EClB7C,IAAAC,GAAA,SAAAC,EAAA,CAA+BC,GAAAF,EAAAC,CAAA,EAC7B,SAAAD,EAAYG,EAAsBC,EAAmD,QACnFH,EAAA,KAAA,IAAA,GAAO,IACT,CAWO,OAAAD,EAAA,UAAA,SAAP,SAAgBK,EAAWC,EAAiB,CAAjB,OAAAA,IAAA,SAAAA,EAAA,GAClB,IACT,EACFN,CAAA,EAjB+BO,EAAY,ECHpC,IAAMC,GAAqC,CAGhD,YAAA,SAAYC,EAAqBC,EAAgB,SAAEC,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,EAAA,CAAA,EAAA,UAAAA,CAAA,EACzC,IAAAC,EAAaL,GAAgB,SACrC,OAAIK,GAAQ,MAARA,EAAU,YACLA,EAAS,YAAW,MAApBA,EAAQC,EAAA,CAAaL,EAASC,CAAO,EAAAK,EAAKJ,CAAI,CAAA,CAAA,EAEhD,YAAW,MAAA,OAAAG,EAAA,CAACL,EAASC,CAAO,EAAAK,EAAKJ,CAAI,CAAA,CAAA,CAC9C,EACA,cAAA,SAAcK,EAAM,CACV,IAAAH,EAAaL,GAAgB,SACrC,QAAQK,GAAQ,KAAA,OAARA,EAAU,gBAAiB,eAAeG,CAAa,CACjE,EACA,SAAU,QCrBZ,IAAAC,GAAA,SAAAC,EAAA,CAAoCC,GAAAF,EAAAC,CAAA,EAOlC,SAAAD,EAAsBG,EAAqCC,EAAmD,CAA9G,IAAAC,EACEJ,EAAA,KAAA,KAAME,EAAWC,CAAI,GAAC,KADF,OAAAC,EAAA,UAAAF,EAAqCE,EAAA,KAAAD,EAFjDC,EAAA,QAAmB,IAI7B,CAEO,OAAAL,EAAA,UAAA,SAAP,SAAgBM,EAAWC,EAAiB,OAC1C,GADyBA,IAAA,SAAAA,EAAA,GACrB,KAAK,OACP,OAAO,KAIT,KAAK,MAAQD,EAEb,IAAME,EAAK,KAAK,GACVL,EAAY,KAAK,UAuBvB,OAAIK,GAAM,OACR,KAAK,GAAK,KAAK,eAAeL,EAAWK,EAAID,CAAK,GAKpD,KAAK,QAAU,GAEf,KAAK,MAAQA,EAEb,KAAK,IAAKE,EAAA,KAAK,MAAE,MAAAA,IAAA,OAAAA,EAAI,KAAK,eAAeN,EAAW,KAAK,GAAII,CAAK,EAE3D,IACT,EAEUP,EAAA,UAAA,eAAV,SAAyBG,EAA2BO,EAAmBH,EAAiB,CAAjB,OAAAA,IAAA,SAAAA,EAAA,GAC9DI,GAAiB,YAAYR,EAAU,MAAM,KAAKA,EAAW,IAAI,EAAGI,CAAK,CAClF,EAEUP,EAAA,UAAA,eAAV,SAAyBY,EAA4BJ,EAAkBD,EAAwB,CAE7F,GAFqEA,IAAA,SAAAA,EAAA,GAEjEA,GAAS,MAAQ,KAAK,QAAUA,GAAS,KAAK,UAAY,GAC5D,OAAOC,EAILA,GAAM,MACRG,GAAiB,cAAcH,CAAE,CAIrC,EAMOR,EAAA,UAAA,QAAP,SAAeM,EAAUC,EAAa,CACpC,GAAI,KAAK,OACP,OAAO,IAAI,MAAM,8BAA8B,EAGjD,KAAK,QAAU,GACf,IAAMM,EAAQ,KAAK,SAASP,EAAOC,CAAK,EACxC,GAAIM,EACF,OAAOA,EACE,KAAK,UAAY,IAAS,KAAK,IAAM,OAc9C,KAAK,GAAK,KAAK,eAAe,KAAK,UAAW,KAAK,GAAI,IAAI,EAE/D,EAEUb,EAAA,UAAA,SAAV,SAAmBM,EAAUQ,EAAc,CACzC,IAAIC,EAAmB,GACnBC,EACJ,GAAI,CACF,KAAK,KAAKV,CAAK,QACRW,EAAG,CACVF,EAAU,GAIVC,EAAaC,GAAQ,IAAI,MAAM,oCAAoC,EAErE,GAAIF,EACF,YAAK,YAAW,EACTC,CAEX,EAEAhB,EAAA,UAAA,YAAA,UAAA,CACE,GAAI,CAAC,KAAK,OAAQ,CACV,IAAAS,EAAoB,KAAlBD,EAAEC,EAAA,GAAEN,EAASM,EAAA,UACbS,EAAYf,EAAS,QAE7B,KAAK,KAAO,KAAK,MAAQ,KAAK,UAAY,KAC1C,KAAK,QAAU,GAEfgB,GAAUD,EAAS,IAAI,EACnBV,GAAM,OACR,KAAK,GAAK,KAAK,eAAeL,EAAWK,EAAI,IAAI,GAGnD,KAAK,MAAQ,KACbP,EAAA,UAAM,YAAW,KAAA,IAAA,EAErB,EACFD,CAAA,EA9IoCoB,EAAM,ECgB1C,IAAAC,GAAA,UAAA,CAGE,SAAAA,EAAoBC,EAAoCC,EAAiC,CAAjCA,IAAA,SAAAA,EAAoBF,EAAU,KAAlE,KAAA,oBAAAC,EAClB,KAAK,IAAMC,CACb,CA6BO,OAAAF,EAAA,UAAA,SAAP,SAAmBG,EAAqDC,EAAmBC,EAAS,CAA5B,OAAAD,IAAA,SAAAA,EAAA,GAC/D,IAAI,KAAK,oBAAuB,KAAMD,CAAI,EAAE,SAASE,EAAOD,CAAK,CAC1E,EAnCcJ,EAAA,IAAoBM,GAAsB,IAoC1DN,GArCA,ECnBA,IAAAO,GAAA,SAAAC,EAAA,CAAoCC,GAAAF,EAAAC,CAAA,EAkBlC,SAAAD,EAAYG,EAAgCC,EAAiC,CAAjCA,IAAA,SAAAA,EAAoBC,GAAU,KAA1E,IAAAC,EACEL,EAAA,KAAA,KAAME,EAAiBC,CAAG,GAAC,KAlBtB,OAAAE,EAAA,QAAmC,CAAA,EAOnCA,EAAA,QAAmB,IAY1B,CAEO,OAAAN,EAAA,UAAA,MAAP,SAAaO,EAAwB,CAC3B,IAAAC,EAAY,KAAI,QAExB,GAAI,KAAK,QAAS,CAChBA,EAAQ,KAAKD,CAAM,EACnB,OAGF,IAAIE,EACJ,KAAK,QAAU,GAEf,EACE,IAAKA,EAAQF,EAAO,QAAQA,EAAO,MAAOA,EAAO,KAAK,EACpD,YAEMA,EAASC,EAAQ,MAAK,GAIhC,GAFA,KAAK,QAAU,GAEXC,EAAO,CACT,KAAQF,EAASC,EAAQ,MAAK,GAC5BD,EAAO,YAAW,EAEpB,MAAME,EAEV,EACFT,CAAA,EAhDoCK,EAAS,EC6CtC,IAAMK,GAAiB,IAAIC,GAAeC,EAAW,EAK/CC,GAAQH,GCjDrB,IAAAI,GAAA,SAAAC,EAAA,CAA6CC,GAAAF,EAAAC,CAAA,EAC3C,SAAAD,EAAsBG,EAA8CC,EAAmD,CAAvH,IAAAC,EACEJ,EAAA,KAAA,KAAME,EAAWC,CAAI,GAAC,KADF,OAAAC,EAAA,UAAAF,EAA8CE,EAAA,KAAAD,GAEpE,CAEU,OAAAJ,EAAA,UAAA,eAAV,SAAyBG,EAAoCG,EAAkBC,EAAiB,CAE9F,OAF6EA,IAAA,SAAAA,EAAA,GAEzEA,IAAU,MAAQA,EAAQ,EACrBN,EAAA,UAAM,eAAc,KAAA,KAACE,EAAWG,EAAIC,CAAK,GAGlDJ,EAAU,QAAQ,KAAK,IAAI,EAIpBA,EAAU,aAAeA,EAAU,WAAaK,GAAuB,sBAAsB,UAAA,CAAM,OAAAL,EAAU,MAAM,MAAS,CAAzB,CAA0B,GACtI,EAEUH,EAAA,UAAA,eAAV,SAAyBG,EAAoCG,EAAkBC,EAAiB,OAI9F,GAJ6EA,IAAA,SAAAA,EAAA,GAIzEA,GAAS,KAAOA,EAAQ,EAAI,KAAK,MAAQ,EAC3C,OAAON,EAAA,UAAM,eAAc,KAAA,KAACE,EAAWG,EAAIC,CAAK,EAK1C,IAAAE,EAAYN,EAAS,QACzBG,GAAM,QAAQI,EAAAD,EAAQA,EAAQ,OAAS,CAAC,KAAC,MAAAC,IAAA,OAAA,OAAAA,EAAE,MAAOJ,IACpDE,GAAuB,qBAAqBF,CAAY,EACxDH,EAAU,WAAa,OAI3B,EACFH,CAAA,EApC6CW,EAAW,ECHxD,IAAAC,GAAA,SAAAC,EAAA,CAA6CC,GAAAF,EAAAC,CAAA,EAA7C,SAAAD,GAAA,+CAkCA,CAjCS,OAAAA,EAAA,UAAA,MAAP,SAAaG,EAAyB,CACpC,KAAK,QAAU,GAUf,IAAMC,EAAU,KAAK,WACrB,KAAK,WAAa,OAEV,IAAAC,EAAY,KAAI,QACpBC,EACJH,EAASA,GAAUE,EAAQ,MAAK,EAEhC,EACE,IAAKC,EAAQH,EAAO,QAAQA,EAAO,MAAOA,EAAO,KAAK,EACpD,aAEMA,EAASE,EAAQ,CAAC,IAAMF,EAAO,KAAOC,GAAWC,EAAQ,MAAK,GAIxE,GAFA,KAAK,QAAU,GAEXC,EAAO,CACT,MAAQH,EAASE,EAAQ,CAAC,IAAMF,EAAO,KAAOC,GAAWC,EAAQ,MAAK,GACpEF,EAAO,YAAW,EAEpB,MAAMG,EAEV,EACFN,CAAA,EAlC6CO,EAAc,ECgCpD,IAAMC,GAA0B,IAAIC,GAAwBC,EAAoB,EC8BhF,IAAMC,EAAQ,IAAIC,EAAkB,SAACC,EAAU,CAAK,OAAAA,EAAW,SAAQ,CAAnB,CAAqB,EC9D1E,SAAUC,GAAYC,EAAU,CACpC,OAAOA,GAASC,EAAWD,EAAM,QAAQ,CAC3C,CCDA,SAASE,GAAQC,EAAQ,CACvB,OAAOA,EAAIA,EAAI,OAAS,CAAC,CAC3B,CAEM,SAAUC,GAAkBC,EAAW,CAC3C,OAAOC,EAAWJ,GAAKG,CAAI,CAAC,EAAIA,EAAK,IAAG,EAAK,MAC/C,CAEM,SAAUE,GAAaF,EAAW,CACtC,OAAOG,GAAYN,GAAKG,CAAI,CAAC,EAAIA,EAAK,IAAG,EAAK,MAChD,CAEM,SAAUI,GAAUJ,EAAaK,EAAoB,CACzD,OAAO,OAAOR,GAAKG,CAAI,GAAM,SAAWA,EAAK,IAAG,EAAMK,CACxD,CClBO,IAAMC,GAAe,SAAIC,EAAM,CAAwB,OAAAA,GAAK,OAAOA,EAAE,QAAW,UAAY,OAAOA,GAAM,UAAlD,ECMxD,SAAUC,GAAUC,EAAU,CAClC,OAAOC,EAAWD,GAAK,KAAA,OAALA,EAAO,IAAI,CAC/B,CCHM,SAAUE,GAAoBC,EAAU,CAC5C,OAAOC,EAAWD,EAAME,EAAiB,CAAC,CAC5C,CCLM,SAAUC,GAAmBC,EAAQ,CACzC,OAAO,OAAO,eAAiBC,EAAWD,GAAG,KAAA,OAAHA,EAAM,OAAO,aAAa,CAAC,CACvE,CCAM,SAAUE,GAAiCC,EAAU,CAEzD,OAAO,IAAI,UACT,iBACEA,IAAU,MAAQ,OAAOA,GAAU,SAAW,oBAAsB,IAAIA,EAAK,KAAG,0HACwC,CAE9H,CCXM,SAAUC,IAAiB,CAC/B,OAAI,OAAO,QAAW,YAAc,CAAC,OAAO,SACnC,aAGF,OAAO,QAChB,CAEO,IAAMC,GAAWD,GAAiB,ECJnC,SAAUE,GAAWC,EAAU,CACnC,OAAOC,EAAWD,GAAK,KAAA,OAALA,EAAQE,EAAe,CAAC,CAC5C,CCHM,SAAiBC,GAAsCC,EAAqC,mGAC1FC,EAASD,EAAe,UAAS,2DAGX,MAAA,CAAA,EAAAE,GAAMD,EAAO,KAAI,CAAE,CAAA,gBAArCE,EAAkBC,EAAA,KAAA,EAAhBC,EAAKF,EAAA,MAAEG,EAAIH,EAAA,KACfG,iBAAA,CAAA,EAAA,CAAA,SACF,MAAA,CAAA,EAAAF,EAAA,KAAA,CAAA,qBAEIC,CAAM,CAAA,SAAZ,MAAA,CAAA,EAAAD,EAAA,KAAA,CAAA,SAAA,OAAAA,EAAA,KAAA,mCAGF,OAAAH,EAAO,YAAW,6BAIhB,SAAUM,GAAwBC,EAAQ,CAG9C,OAAOC,EAAWD,GAAG,KAAA,OAAHA,EAAK,SAAS,CAClC,CCPM,SAAUE,EAAaC,EAAyB,CACpD,GAAIA,aAAiBC,EACnB,OAAOD,EAET,GAAIA,GAAS,KAAM,CACjB,GAAIE,GAAoBF,CAAK,EAC3B,OAAOG,GAAsBH,CAAK,EAEpC,GAAII,GAAYJ,CAAK,EACnB,OAAOK,GAAcL,CAAK,EAE5B,GAAIM,GAAUN,CAAK,EACjB,OAAOO,GAAYP,CAAK,EAE1B,GAAIQ,GAAgBR,CAAK,EACvB,OAAOS,GAAkBT,CAAK,EAEhC,GAAIU,GAAWV,CAAK,EAClB,OAAOW,GAAaX,CAAK,EAE3B,GAAIY,GAAqBZ,CAAK,EAC5B,OAAOa,GAAuBb,CAAK,EAIvC,MAAMc,GAAiCd,CAAK,CAC9C,CAMM,SAAUG,GAAyBY,EAAQ,CAC/C,OAAO,IAAId,EAAW,SAACe,EAAyB,CAC9C,IAAMC,EAAMF,EAAIG,EAAiB,EAAC,EAClC,GAAIC,EAAWF,EAAI,SAAS,EAC1B,OAAOA,EAAI,UAAUD,CAAU,EAGjC,MAAM,IAAI,UAAU,gEAAgE,CACtF,CAAC,CACH,CASM,SAAUX,GAAiBe,EAAmB,CAClD,OAAO,IAAInB,EAAW,SAACe,EAAyB,CAU9C,QAASK,EAAI,EAAGA,EAAID,EAAM,QAAU,CAACJ,EAAW,OAAQK,IACtDL,EAAW,KAAKI,EAAMC,CAAC,CAAC,EAE1BL,EAAW,SAAQ,CACrB,CAAC,CACH,CAEM,SAAUT,GAAee,EAAuB,CACpD,OAAO,IAAIrB,EAAW,SAACe,EAAyB,CAC9CM,EACG,KACC,SAACC,EAAK,CACCP,EAAW,SACdA,EAAW,KAAKO,CAAK,EACrBP,EAAW,SAAQ,EAEvB,EACA,SAACQ,EAAQ,CAAK,OAAAR,EAAW,MAAMQ,CAAG,CAApB,CAAqB,EAEpC,KAAK,KAAMC,EAAoB,CACpC,CAAC,CACH,CAEM,SAAUd,GAAgBe,EAAqB,CACnD,OAAO,IAAIzB,EAAW,SAACe,EAAyB,aAC9C,QAAoBW,EAAAC,GAAAF,CAAQ,EAAAG,EAAAF,EAAA,KAAA,EAAA,CAAAE,EAAA,KAAAA,EAAAF,EAAA,KAAA,EAAE,CAAzB,IAAMJ,EAAKM,EAAA,MAEd,GADAb,EAAW,KAAKO,CAAK,EACjBP,EAAW,OACb,yGAGJA,EAAW,SAAQ,CACrB,CAAC,CACH,CAEM,SAAUP,GAAqBqB,EAA+B,CAClE,OAAO,IAAI7B,EAAW,SAACe,EAAyB,CAC9Ce,GAAQD,EAAed,CAAU,EAAE,MAAM,SAACQ,EAAG,CAAK,OAAAR,EAAW,MAAMQ,CAAG,CAApB,CAAqB,CACzE,CAAC,CACH,CAEM,SAAUX,GAA0BmB,EAAqC,CAC7E,OAAOvB,GAAkBwB,GAAmCD,CAAc,CAAC,CAC7E,CAEA,SAAeD,GAAWD,EAAiCd,EAAyB,uIACxDkB,EAAAC,GAAAL,CAAa,gFAIrC,GAJeP,EAAKa,EAAA,MACpBpB,EAAW,KAAKO,CAAK,EAGjBP,EAAW,OACb,MAAA,CAAA,CAAA,6RAGJ,OAAAA,EAAW,SAAQ,WChHf,SAAUqB,GACdC,EACAC,EACAC,EACAC,EACAC,EAAc,CADdD,IAAA,SAAAA,EAAA,GACAC,IAAA,SAAAA,EAAA,IAEA,IAAMC,EAAuBJ,EAAU,SAAS,UAAA,CAC9CC,EAAI,EACAE,EACFJ,EAAmB,IAAI,KAAK,SAAS,KAAMG,CAAK,CAAC,EAEjD,KAAK,YAAW,CAEpB,EAAGA,CAAK,EAIR,GAFAH,EAAmB,IAAIK,CAAoB,EAEvC,CAACD,EAKH,OAAOC,CAEX,CCeM,SAAUC,GAAaC,EAA0BC,EAAS,CAAT,OAAAA,IAAA,SAAAA,EAAA,GAC9CC,EAAQ,SAACC,EAAQC,EAAU,CAChCD,EAAO,UACLE,EACED,EACA,SAACE,EAAK,CAAK,OAAAC,GAAgBH,EAAYJ,EAAW,UAAA,CAAM,OAAAI,EAAW,KAAKE,CAAK,CAArB,EAAwBL,CAAK,CAA1E,EACX,UAAA,CAAM,OAAAM,GAAgBH,EAAYJ,EAAW,UAAA,CAAM,OAAAI,EAAW,SAAQ,CAAnB,EAAuBH,CAAK,CAAzE,EACN,SAACO,EAAG,CAAK,OAAAD,GAAgBH,EAAYJ,EAAW,UAAA,CAAM,OAAAI,EAAW,MAAMI,CAAG,CAApB,EAAuBP,CAAK,CAAzE,CAA0E,CACpF,CAEL,CAAC,CACH,CCPM,SAAUQ,GAAeC,EAA0BC,EAAiB,CAAjB,OAAAA,IAAA,SAAAA,EAAA,GAChDC,EAAQ,SAACC,EAAQC,EAAU,CAChCA,EAAW,IAAIJ,EAAU,SAAS,UAAA,CAAM,OAAAG,EAAO,UAAUC,CAAU,CAA3B,EAA8BH,CAAK,CAAC,CAC9E,CAAC,CACH,CC7DM,SAAUI,GAAsBC,EAA6BC,EAAwB,CACzF,OAAOC,EAAUF,CAAK,EAAE,KAAKG,GAAYF,CAAS,EAAGG,GAAUH,CAAS,CAAC,CAC3E,CCFM,SAAUI,GAAmBC,EAAuBC,EAAwB,CAChF,OAAOC,EAAUF,CAAK,EAAE,KAAKG,GAAYF,CAAS,EAAGG,GAAUH,CAAS,CAAC,CAC3E,CCJM,SAAUI,GAAiBC,EAAqBC,EAAwB,CAC5E,OAAO,IAAIC,EAAc,SAACC,EAAU,CAElC,IAAIC,EAAI,EAER,OAAOH,EAAU,SAAS,UAAA,CACpBG,IAAMJ,EAAM,OAGdG,EAAW,SAAQ,GAInBA,EAAW,KAAKH,EAAMI,GAAG,CAAC,EAIrBD,EAAW,QACd,KAAK,SAAQ,EAGnB,CAAC,CACH,CAAC,CACH,CCfM,SAAUE,GAAoBC,EAAoBC,EAAwB,CAC9E,OAAO,IAAIC,EAAc,SAACC,EAAU,CAClC,IAAIC,EAKJ,OAAAC,GAAgBF,EAAYF,EAAW,UAAA,CAErCG,EAAYJ,EAAcI,EAAe,EAAC,EAE1CC,GACEF,EACAF,EACA,UAAA,OACMK,EACAC,EACJ,GAAI,CAEDC,EAAkBJ,EAAS,KAAI,EAA7BE,EAAKE,EAAA,MAAED,EAAIC,EAAA,WACPC,EAAK,CAEZN,EAAW,MAAMM,CAAG,EACpB,OAGEF,EAKFJ,EAAW,SAAQ,EAGnBA,EAAW,KAAKG,CAAK,CAEzB,EACA,EACA,EAAI,CAER,CAAC,EAMM,UAAA,CAAM,OAAAI,EAAWN,GAAQ,KAAA,OAARA,EAAU,MAAM,GAAKA,EAAS,OAAM,CAA/C,CACf,CAAC,CACH,CCvDM,SAAUO,GAAyBC,EAAyBC,EAAwB,CACxF,GAAI,CAACD,EACH,MAAM,IAAI,MAAM,yBAAyB,EAE3C,OAAO,IAAIE,EAAc,SAACC,EAAU,CAClCC,GAAgBD,EAAYF,EAAW,UAAA,CACrC,IAAMI,EAAWL,EAAM,OAAO,aAAa,EAAC,EAC5CI,GACED,EACAF,EACA,UAAA,CACEI,EAAS,KAAI,EAAG,KAAK,SAACC,EAAM,CACtBA,EAAO,KAGTH,EAAW,SAAQ,EAEnBA,EAAW,KAAKG,EAAO,KAAK,CAEhC,CAAC,CACH,EACA,EACA,EAAI,CAER,CAAC,CACH,CAAC,CACH,CCzBM,SAAUC,GAA8BC,EAA8BC,EAAwB,CAClG,OAAOC,GAAsBC,GAAmCH,CAAK,EAAGC,CAAS,CACnF,CCoBM,SAAUG,GAAaC,EAA2BC,EAAwB,CAC9E,GAAID,GAAS,KAAM,CACjB,GAAIE,GAAoBF,CAAK,EAC3B,OAAOG,GAAmBH,EAAOC,CAAS,EAE5C,GAAIG,GAAYJ,CAAK,EACnB,OAAOK,GAAcL,EAAOC,CAAS,EAEvC,GAAIK,GAAUN,CAAK,EACjB,OAAOO,GAAgBP,EAAOC,CAAS,EAEzC,GAAIO,GAAgBR,CAAK,EACvB,OAAOS,GAAsBT,EAAOC,CAAS,EAE/C,GAAIS,GAAWV,CAAK,EAClB,OAAOW,GAAiBX,EAAOC,CAAS,EAE1C,GAAIW,GAAqBZ,CAAK,EAC5B,OAAOa,GAA2Bb,EAAOC,CAAS,EAGtD,MAAMa,GAAiCd,CAAK,CAC9C,CCoDM,SAAUe,GAAQC,EAA2BC,EAAyB,CAC1E,OAAOA,EAAYC,GAAUF,EAAOC,CAAS,EAAIE,EAAUH,CAAK,CAClE,CCxBM,SAAUI,GAAE,SAAIC,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,CAAA,EAAA,UAAAA,CAAA,EACpB,IAAMC,EAAYC,GAAaH,CAAI,EACnC,OAAOI,GAAKJ,EAAaE,CAAS,CACpC,CCsCM,SAAUG,GAAWC,EAA0BC,EAAyB,CAC5E,IAAMC,EAAeC,EAAWH,CAAmB,EAAIA,EAAsB,UAAA,CAAM,OAAAA,CAAA,EAC7EI,EAAO,SAACC,EAA6B,CAAK,OAAAA,EAAW,MAAMH,EAAY,CAAE,CAA/B,EAChD,OAAO,IAAII,EAAWL,EAAY,SAACI,EAAU,CAAK,OAAAJ,EAAU,SAASG,EAAa,EAAGC,CAAU,CAA7C,EAAiDD,CAAI,CACzG,CCpGO,IAAMG,GAA6BC,GAAiB,SAACC,EAAM,CAAK,OAAA,UAAuB,CAC5FA,EAAO,IAAI,EACX,KAAK,KAAO,aACZ,KAAK,QAAU,yBACjB,CAJuE,CAItE,ECrBK,SAAUC,GAAYC,EAAU,CACpC,OAAOA,aAAiB,MAAQ,CAAC,MAAMA,CAAY,CACrD,CCsCM,SAAUC,EAAUC,EAAyCC,EAAa,CAC9E,OAAOC,EAAQ,SAACC,EAAQC,EAAU,CAEhC,IAAIC,EAAQ,EAGZF,EAAO,UACLG,EAAyBF,EAAY,SAACG,EAAQ,CAG5CH,EAAW,KAAKJ,EAAQ,KAAKC,EAASM,EAAOF,GAAO,CAAC,CACvD,CAAC,CAAC,CAEN,CAAC,CACH,CC1DQ,IAAAG,GAAY,MAAK,QAEzB,SAASC,GAAkBC,EAA6BC,EAAW,CAC/D,OAAOH,GAAQG,CAAI,EAAID,EAAE,MAAA,OAAAE,EAAA,CAAA,EAAAC,EAAIF,CAAI,CAAA,CAAA,EAAID,EAAGC,CAAI,CAChD,CAMM,SAAUG,GAAuBJ,EAA2B,CAC9D,OAAOK,EAAI,SAAAJ,EAAI,CAAI,OAAAF,GAAYC,EAAIC,CAAI,CAApB,CAAqB,CAC5C,CCfQ,IAAAK,GAAY,MAAK,QACjBC,GAA0D,OAAM,eAArCC,GAA+B,OAAM,UAAlBC,GAAY,OAAM,KAQlE,SAAUC,GAAqDC,EAAuB,CAC1F,GAAIA,EAAK,SAAW,EAAG,CACrB,IAAMC,EAAQD,EAAK,CAAC,EACpB,GAAIL,GAAQM,CAAK,EACf,MAAO,CAAE,KAAMA,EAAO,KAAM,IAAI,EAElC,GAAIC,GAAOD,CAAK,EAAG,CACjB,IAAME,EAAOL,GAAQG,CAAK,EAC1B,MAAO,CACL,KAAME,EAAK,IAAI,SAACC,EAAG,CAAK,OAAAH,EAAMG,CAAG,CAAT,CAAU,EAClC,KAAID,IAKV,MAAO,CAAE,KAAMH,EAAa,KAAM,IAAI,CACxC,CAEA,SAASE,GAAOG,EAAQ,CACtB,OAAOA,GAAO,OAAOA,GAAQ,UAAYT,GAAeS,CAAG,IAAMR,EACnE,CC7BM,SAAUS,GAAaC,EAAgBC,EAAa,CACxD,OAAOD,EAAK,OAAO,SAACE,EAAQC,EAAKC,EAAC,CAAK,OAAEF,EAAOC,CAAG,EAAIF,EAAOG,CAAC,EAAIF,CAA5B,EAAqC,CAAA,CAAS,CACvF,CCsMM,SAAUG,GAAa,SAAoCC,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,CAAA,EAAA,UAAAA,CAAA,EAC/D,IAAMC,EAAYC,GAAaH,CAAI,EAC7BI,EAAiBC,GAAkBL,CAAI,EAEvCM,EAA8BC,GAAqBP,CAAI,EAA/CQ,EAAWF,EAAA,KAAEG,EAAIH,EAAA,KAE/B,GAAIE,EAAY,SAAW,EAIzB,OAAOE,GAAK,CAAA,EAAIR,CAAgB,EAGlC,IAAMS,EAAS,IAAIC,EACjBC,GACEL,EACAN,EACAO,EAEI,SAACK,EAAM,CAAK,OAAAC,GAAaN,EAAMK,CAAM,CAAzB,EAEZE,EAAQ,CACb,EAGH,OAAOZ,EAAkBO,EAAO,KAAKM,GAAiBb,CAAc,CAAC,EAAsBO,CAC7F,CAEM,SAAUE,GACdL,EACAN,EACAgB,EAAiD,CAAjD,OAAAA,IAAA,SAAAA,EAAAF,IAEO,SAACG,EAA2B,CAGjCC,GACElB,EACA,UAAA,CAaE,QAZQmB,EAAWb,EAAW,OAExBM,EAAS,IAAI,MAAMO,CAAM,EAG3BC,EAASD,EAITE,EAAuBF,aAGlBG,EAAC,CACRJ,GACElB,EACA,UAAA,CACE,IAAMuB,EAASf,GAAKF,EAAYgB,CAAC,EAAGtB,CAAgB,EAChDwB,EAAgB,GACpBD,EAAO,UACLE,EACER,EACA,SAACS,EAAK,CAEJd,EAAOU,CAAC,EAAII,EACPF,IAEHA,EAAgB,GAChBH,KAEGA,GAGHJ,EAAW,KAAKD,EAAeJ,EAAO,MAAK,CAAE,CAAC,CAElD,EACA,UAAA,CACO,EAAEQ,GAGLH,EAAW,SAAQ,CAEvB,CAAC,CACF,CAEL,EACAA,CAAU,GAjCLK,EAAI,EAAGA,EAAIH,EAAQG,MAAnBA,CAAC,CAoCZ,EACAL,CAAU,CAEd,CACF,CAMA,SAASC,GAAclB,EAAsC2B,EAAqBC,EAA0B,CACtG5B,EACF6B,GAAgBD,EAAc5B,EAAW2B,CAAO,EAEhDA,EAAO,CAEX,CC3RM,SAAUG,GACdC,EACAC,EACAC,EACAC,EACAC,EACAC,EACAC,EACAC,EAAgC,CAGhC,IAAMC,EAAc,CAAA,EAEhBC,EAAS,EAETC,EAAQ,EAERC,EAAa,GAKXC,EAAgB,UAAA,CAIhBD,GAAc,CAACH,EAAO,QAAU,CAACC,GACnCR,EAAW,SAAQ,CAEvB,EAGMY,EAAY,SAACC,EAAQ,CAAK,OAACL,EAASN,EAAaY,EAAWD,CAAK,EAAIN,EAAO,KAAKM,CAAK,CAA5D,EAE1BC,EAAa,SAACD,EAAQ,CAI1BT,GAAUJ,EAAW,KAAKa,CAAY,EAItCL,IAKA,IAAIO,EAAgB,GAGpBC,EAAUf,EAAQY,EAAOJ,GAAO,CAAC,EAAE,UACjCQ,EACEjB,EACA,SAACkB,GAAU,CAGTf,GAAY,MAAZA,EAAee,EAAU,EAErBd,EAGFQ,EAAUM,EAAiB,EAG3BlB,EAAW,KAAKkB,EAAU,CAE9B,EACA,UAAA,CAGEH,EAAgB,EAClB,EAEA,OACA,UAAA,CAIE,GAAIA,EAKF,GAAI,CAIFP,IAKA,sBACE,IAAMW,EAAgBZ,EAAO,MAAK,EAI9BF,EACFe,GAAgBpB,EAAYK,EAAmB,UAAA,CAAM,OAAAS,EAAWK,CAAa,CAAxB,CAAyB,EAE9EL,EAAWK,CAAa,GARrBZ,EAAO,QAAUC,EAASN,QAYjCS,EAAa,QACNU,EAAK,CACZrB,EAAW,MAAMqB,CAAG,EAG1B,CAAC,CACF,CAEL,EAGA,OAAAtB,EAAO,UACLkB,EAAyBjB,EAAYY,EAAW,UAAA,CAE9CF,EAAa,GACbC,EAAa,CACf,CAAC,CAAC,EAKG,UAAA,CACLL,GAAmB,MAAnBA,EAAmB,CACrB,CACF,CClEM,SAAUgB,GACdC,EACAC,EACAC,EAA6B,CAE7B,OAFAA,IAAA,SAAAA,EAAA,KAEIC,EAAWF,CAAc,EAEpBF,GAAS,SAACK,EAAGC,EAAC,CAAK,OAAAC,EAAI,SAACC,EAAQC,EAAU,CAAK,OAAAP,EAAeG,EAAGG,EAAGF,EAAGG,CAAE,CAA1B,CAA2B,EAAEC,EAAUT,EAAQI,EAAGC,CAAC,CAAC,CAAC,CAAjF,EAAoFH,CAAU,GAC/G,OAAOD,GAAmB,WACnCC,EAAaD,GAGRS,EAAQ,SAACC,EAAQC,EAAU,CAAK,OAAAC,GAAeF,EAAQC,EAAYZ,EAASE,CAAU,CAAtD,CAAuD,EAChG,CChCM,SAAUY,GAAyCC,EAA6B,CAA7B,OAAAA,IAAA,SAAAA,EAAA,KAChDC,GAASC,GAAUF,CAAU,CACtC,CCNM,SAAUG,IAAS,CACvB,OAAOC,GAAS,CAAC,CACnB,CCmDM,SAAUC,IAAM,SAACC,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,CAAA,EAAA,UAAAA,CAAA,EACrB,OAAOC,GAAS,EAAGC,GAAKH,EAAMI,GAAaJ,CAAI,CAAC,CAAC,CACnD,CC9DM,SAAUK,EAAsCC,EAA0B,CAC9E,OAAO,IAAIC,EAA+B,SAACC,EAAU,CACnDC,EAAUH,EAAiB,CAAE,EAAE,UAAUE,CAAU,CACrD,CAAC,CACH,CChDA,IAAME,GAA0B,CAAC,cAAe,gBAAgB,EAC1DC,GAAqB,CAAC,mBAAoB,qBAAqB,EAC/DC,GAAgB,CAAC,KAAM,KAAK,EAkO5B,SAAUC,EACdC,EACAC,EACAC,EACAC,EAAsC,CAMtC,GAJIC,EAAWF,CAAO,IACpBC,EAAiBD,EACjBA,EAAU,QAERC,EACF,OAAOJ,EAAaC,EAAQC,EAAWC,CAA+B,EAAE,KAAKG,GAAiBF,CAAc,CAAC,EAUzG,IAAAG,EAAAC,EAEJC,GAAcR,CAAM,EAChBH,GAAmB,IAAI,SAACY,EAAU,CAAK,OAAA,SAACC,EAAY,CAAK,OAAAV,EAAOS,CAAU,EAAER,EAAWS,EAASR,CAA+B,CAAtE,CAAlB,CAAyF,EAElIS,GAAwBX,CAAM,EAC5BJ,GAAwB,IAAIgB,GAAwBZ,EAAQC,CAAS,CAAC,EACtEY,GAA0Bb,CAAM,EAChCF,GAAc,IAAIc,GAAwBZ,EAAQC,CAAS,CAAC,EAC5D,CAAA,EAAE,CAAA,EATDa,EAAGR,EAAA,CAAA,EAAES,EAAMT,EAAA,CAAA,EAgBlB,GAAI,CAACQ,GACCE,GAAYhB,CAAM,EACpB,OAAOiB,GAAS,SAACC,EAAc,CAAK,OAAAnB,EAAUmB,EAAWjB,EAAWC,CAA+B,CAA/D,CAAgE,EAClGiB,EAAUnB,CAAM,CAAC,EAOvB,GAAI,CAACc,EACH,MAAM,IAAI,UAAU,sBAAsB,EAG5C,OAAO,IAAIM,EAAc,SAACC,EAAU,CAIlC,IAAMX,EAAU,UAAA,SAACY,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,CAAA,EAAA,UAAAA,CAAA,EAAmB,OAAAF,EAAW,KAAK,EAAIC,EAAK,OAASA,EAAOA,EAAK,CAAC,CAAC,CAAhD,EAEpC,OAAAR,EAAIJ,CAAO,EAEJ,UAAA,CAAM,OAAAK,EAAQL,CAAO,CAAf,CACf,CAAC,CACH,CASA,SAASE,GAAwBZ,EAAaC,EAAiB,CAC7D,OAAO,SAACQ,EAAkB,CAAK,OAAA,SAACC,EAAY,CAAK,OAAAV,EAAOS,CAAU,EAAER,EAAWS,CAAO,CAArC,CAAlB,CACjC,CAOA,SAASC,GAAwBX,EAAW,CAC1C,OAAOI,EAAWJ,EAAO,WAAW,GAAKI,EAAWJ,EAAO,cAAc,CAC3E,CAOA,SAASa,GAA0Bb,EAAW,CAC5C,OAAOI,EAAWJ,EAAO,EAAE,GAAKI,EAAWJ,EAAO,GAAG,CACvD,CAOA,SAASQ,GAAcR,EAAW,CAChC,OAAOI,EAAWJ,EAAO,gBAAgB,GAAKI,EAAWJ,EAAO,mBAAmB,CACrF,CCnMM,SAAUwB,GACdC,EACAC,EACAC,EAAsC,CAEtC,OAAIA,EACKH,GAAoBC,EAAYC,CAAa,EAAE,KAAKE,GAAiBD,CAAc,CAAC,EAGtF,IAAIE,EAAoB,SAACC,EAAU,CACxC,IAAMC,EAAU,UAAA,SAACC,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,CAAA,EAAA,UAAAA,CAAA,EAAc,OAAAH,EAAW,KAAKE,EAAE,SAAW,EAAIA,EAAE,CAAC,EAAIA,CAAC,CAAzC,EACzBE,EAAWT,EAAWM,CAAO,EACnC,OAAOI,EAAWT,CAAa,EAAI,UAAA,CAAM,OAAAA,EAAcK,EAASG,CAAQ,CAA/B,EAAmC,MAC9E,CAAC,CACH,CCtBM,SAAUE,GACdC,EACAC,EACAC,EAAyC,CAFzCF,IAAA,SAAAA,EAAA,GAEAE,IAAA,SAAAA,EAAAC,IAIA,IAAIC,EAAmB,GAEvB,OAAIH,GAAuB,OAIrBI,GAAYJ,CAAmB,EACjCC,EAAYD,EAIZG,EAAmBH,GAIhB,IAAIK,EAAW,SAACC,EAAU,CAI/B,IAAIC,EAAMC,GAAYT,CAAO,EAAI,CAACA,EAAUE,EAAW,IAAG,EAAKF,EAE3DQ,EAAM,IAERA,EAAM,GAIR,IAAIE,EAAI,EAGR,OAAOR,EAAU,SAAS,UAAA,CACnBK,EAAW,SAEdA,EAAW,KAAKG,GAAG,EAEf,GAAKN,EAGP,KAAK,SAAS,OAAWA,CAAgB,EAGzCG,EAAW,SAAQ,EAGzB,EAAGC,CAAG,CACR,CAAC,CACH,CChGM,SAAUG,GAAK,SAACC,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,CAAA,EAAA,UAAAA,CAAA,EACpB,IAAMC,EAAYC,GAAaH,CAAI,EAC7BI,EAAaC,GAAUL,EAAM,GAAQ,EACrCM,EAAUN,EAChB,OAAQM,EAAQ,OAGZA,EAAQ,SAAW,EAEnBC,EAAUD,EAAQ,CAAC,CAAC,EAEpBE,GAASJ,CAAU,EAAEK,GAAKH,EAASJ,CAAS,CAAC,EAL7CQ,CAMN,CCjEO,IAAMC,GAAQ,IAAIC,EAAkBC,EAAI,ECpCvC,IAAAC,GAAY,MAAK,QAMnB,SAAUC,GAAkBC,EAAiB,CACjD,OAAOA,EAAK,SAAW,GAAKF,GAAQE,EAAK,CAAC,CAAC,EAAIA,EAAK,CAAC,EAAKA,CAC5D,CCoDM,SAAUC,EAAUC,EAAiDC,EAAa,CACtF,OAAOC,EAAQ,SAACC,EAAQC,EAAU,CAEhC,IAAIC,EAAQ,EAIZF,EAAO,UAILG,EAAyBF,EAAY,SAACG,EAAK,CAAK,OAAAP,EAAU,KAAKC,EAASM,EAAOF,GAAO,GAAKD,EAAW,KAAKG,CAAK,CAAhE,CAAiE,CAAC,CAEtH,CAAC,CACH,CCxBM,SAAUC,IAAG,SAACC,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,CAAA,EAAA,UAAAA,CAAA,EAClB,IAAMC,EAAiBC,GAAkBH,CAAI,EAEvCI,EAAUC,GAAeL,CAAI,EAEnC,OAAOI,EAAQ,OACX,IAAIE,EAAsB,SAACC,EAAU,CAGnC,IAAIC,EAAuBJ,EAAQ,IAAI,UAAA,CAAM,MAAA,CAAA,CAAA,CAAE,EAK3CK,EAAYL,EAAQ,IAAI,UAAA,CAAM,MAAA,EAAA,CAAK,EAGvCG,EAAW,IAAI,UAAA,CACbC,EAAUC,EAAY,IACxB,CAAC,EAKD,mBAASC,EAAW,CAClBC,EAAUP,EAAQM,CAAW,CAAC,EAAE,UAC9BE,EACEL,EACA,SAACM,EAAK,CAKJ,GAJAL,EAAQE,CAAW,EAAE,KAAKG,CAAK,EAI3BL,EAAQ,MAAM,SAACM,EAAM,CAAK,OAAAA,EAAO,MAAP,CAAa,EAAG,CAC5C,IAAMC,EAAcP,EAAQ,IAAI,SAACM,EAAM,CAAK,OAAAA,EAAO,MAAK,CAAZ,CAAe,EAE3DP,EAAW,KAAKL,EAAiBA,EAAc,MAAA,OAAAc,EAAA,CAAA,EAAAC,EAAIF,CAAM,CAAA,CAAA,EAAIA,CAAM,EAI/DP,EAAQ,KAAK,SAACM,EAAQI,EAAC,CAAK,MAAA,CAACJ,EAAO,QAAUL,EAAUS,CAAC,CAA7B,CAA8B,GAC5DX,EAAW,SAAQ,EAGzB,EACA,UAAA,CAGEE,EAAUC,CAAW,EAAI,GAIzB,CAACF,EAAQE,CAAW,EAAE,QAAUH,EAAW,SAAQ,CACrD,CAAC,CACF,GA9BIG,EAAc,EAAG,CAACH,EAAW,QAAUG,EAAcN,EAAQ,OAAQM,MAArEA,CAAW,EAmCpB,OAAO,UAAA,CACLF,EAAUC,EAAY,IACxB,CACF,CAAC,EACDU,CACN,CC9DM,SAAUC,GAASC,EAAoD,CAC3E,OAAOC,EAAQ,SAACC,EAAQC,EAAU,CAChC,IAAIC,EAAW,GACXC,EAAsB,KACtBC,EAA6C,KAC7CC,EAAa,GAEXC,EAAc,UAAA,CAGlB,GAFAF,GAAkB,MAAlBA,EAAoB,YAAW,EAC/BA,EAAqB,KACjBF,EAAU,CACZA,EAAW,GACX,IAAMK,EAAQJ,EACdA,EAAY,KACZF,EAAW,KAAKM,CAAK,EAEvBF,GAAcJ,EAAW,SAAQ,CACnC,EAEMO,EAAkB,UAAA,CACtBJ,EAAqB,KACrBC,GAAcJ,EAAW,SAAQ,CACnC,EAEAD,EAAO,UACLS,EACER,EACA,SAACM,EAAK,CACJL,EAAW,GACXC,EAAYI,EACPH,GACHM,EAAUZ,EAAiBS,CAAK,CAAC,EAAE,UAChCH,EAAqBK,EAAyBR,EAAYK,EAAaE,CAAe,CAAE,CAG/F,EACA,UAAA,CACEH,EAAa,IACZ,CAACH,GAAY,CAACE,GAAsBA,EAAmB,SAAWH,EAAW,SAAQ,CACxF,CAAC,CACF,CAEL,CAAC,CACH,CC3CM,SAAUU,GAAaC,EAAkBC,EAAyC,CAAzC,OAAAA,IAAA,SAAAA,EAAAC,IACtCC,GAAM,UAAA,CAAM,OAAAC,GAAMJ,EAAUC,CAAS,CAAzB,CAA0B,CAC/C,CCEM,SAAUI,GAAeC,EAAoBC,EAAsC,CAAtC,OAAAA,IAAA,SAAAA,EAAA,MAGjDA,EAAmBA,GAAgB,KAAhBA,EAAoBD,EAEhCE,EAAQ,SAACC,EAAQC,EAAU,CAChC,IAAIC,EAAiB,CAAA,EACjBC,EAAQ,EAEZH,EAAO,UACLI,EACEH,EACA,SAACI,EAAK,aACAC,EAAuB,KAKvBH,IAAUL,IAAsB,GAClCI,EAAQ,KAAK,CAAA,CAAE,MAIjB,QAAqBK,EAAAC,GAAAN,CAAO,EAAAO,EAAAF,EAAA,KAAA,EAAA,CAAAE,EAAA,KAAAA,EAAAF,EAAA,KAAA,EAAE,CAAzB,IAAMG,EAAMD,EAAA,MACfC,EAAO,KAAKL,CAAK,EAMbR,GAAca,EAAO,SACvBJ,EAASA,GAAM,KAANA,EAAU,CAAA,EACnBA,EAAO,KAAKI,CAAM,uGAItB,GAAIJ,MAIF,QAAqBK,EAAAH,GAAAF,CAAM,EAAAM,EAAAD,EAAA,KAAA,EAAA,CAAAC,EAAA,KAAAA,EAAAD,EAAA,KAAA,EAAE,CAAxB,IAAMD,EAAME,EAAA,MACfC,GAAUX,EAASQ,CAAM,EACzBT,EAAW,KAAKS,CAAM,sGAG5B,EACA,UAAA,aAGE,QAAqBI,EAAAN,GAAAN,CAAO,EAAAa,EAAAD,EAAA,KAAA,EAAA,CAAAC,EAAA,KAAAA,EAAAD,EAAA,KAAA,EAAE,CAAzB,IAAMJ,EAAMK,EAAA,MACfd,EAAW,KAAKS,CAAM,oGAExBT,EAAW,SAAQ,CACrB,EAEA,OACA,UAAA,CAEEC,EAAU,IACZ,CAAC,CACF,CAEL,CAAC,CACH,CCbM,SAAUc,GACdC,EAAgD,CAEhD,OAAOC,EAAQ,SAACC,EAAQC,EAAU,CAChC,IAAIC,EAAgC,KAChCC,EAAY,GACZC,EAEJF,EAAWF,EAAO,UAChBK,EAAyBJ,EAAY,OAAW,OAAW,SAACK,EAAG,CAC7DF,EAAgBG,EAAUT,EAASQ,EAAKT,GAAWC,CAAQ,EAAEE,CAAM,CAAC,CAAC,EACjEE,GACFA,EAAS,YAAW,EACpBA,EAAW,KACXE,EAAc,UAAUH,CAAU,GAIlCE,EAAY,EAEhB,CAAC,CAAC,EAGAA,IAMFD,EAAS,YAAW,EACpBA,EAAW,KACXE,EAAe,UAAUH,CAAU,EAEvC,CAAC,CACH,CC/HM,SAAUO,GACdC,EACAC,EACAC,EACAC,EACAC,EAAqC,CAErC,OAAO,SAACC,EAAuBC,EAA2B,CAIxD,IAAIC,EAAWL,EAIXM,EAAaP,EAEbQ,EAAQ,EAGZJ,EAAO,UACLK,EACEJ,EACA,SAACK,EAAK,CAEJ,IAAMC,EAAIH,IAEVD,EAAQD,EAEJP,EAAYQ,EAAOG,EAAOC,CAAC,GAIzBL,EAAW,GAAOI,GAGxBR,GAAcG,EAAW,KAAKE,CAAK,CACrC,EAGAJ,GACG,UAAA,CACCG,GAAYD,EAAW,KAAKE,CAAK,EACjCF,EAAW,SAAQ,CACrB,CAAE,CACL,CAEL,CACF,CCnCM,SAAUO,IAAa,SAAOC,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,CAAA,EAAA,UAAAA,CAAA,EAClC,IAAMC,EAAiBC,GAAkBH,CAAI,EAC7C,OAAOE,EACHE,GAAKL,GAAa,MAAA,OAAAM,EAAA,CAAA,EAAAC,EAAKN,CAAoC,CAAA,CAAA,EAAGO,GAAiBL,CAAc,CAAC,EAC9FM,EAAQ,SAACC,EAAQC,EAAU,CACzBC,GAAiBN,EAAA,CAAEI,CAAM,EAAAH,EAAKM,GAAeZ,CAAI,CAAC,CAAA,CAAA,EAAGU,CAAU,CACjE,CAAC,CACP,CCUM,SAAUG,IAAiB,SAC/BC,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,CAAA,EAAA,UAAAA,CAAA,EAEA,OAAOC,GAAa,MAAA,OAAAC,EAAA,CAAA,EAAAC,EAAIJ,CAAY,CAAA,CAAA,CACtC,CCgBM,SAAUK,GAAgBC,EAAiBC,EAAyC,CAAzC,OAAAA,IAAA,SAAAA,EAAAC,IACxCC,EAAQ,SAACC,EAAQC,EAAU,CAChC,IAAIC,EAAkC,KAClCC,EAAsB,KACtBC,EAA0B,KAExBC,EAAO,UAAA,CACX,GAAIH,EAAY,CAEdA,EAAW,YAAW,EACtBA,EAAa,KACb,IAAMI,EAAQH,EACdA,EAAY,KACZF,EAAW,KAAKK,CAAK,EAEzB,EACA,SAASC,GAAY,CAInB,IAAMC,EAAaJ,EAAYR,EACzBa,EAAMZ,EAAU,IAAG,EACzB,GAAIY,EAAMD,EAAY,CAEpBN,EAAa,KAAK,SAAS,OAAWM,EAAaC,CAAG,EACtDR,EAAW,IAAIC,CAAU,EACzB,OAGFG,EAAI,CACN,CAEAL,EAAO,UACLU,EACET,EACA,SAACK,EAAQ,CACPH,EAAYG,EACZF,EAAWP,EAAU,IAAG,EAGnBK,IACHA,EAAaL,EAAU,SAASU,EAAcX,CAAO,EACrDK,EAAW,IAAIC,CAAU,EAE7B,EACA,UAAA,CAGEG,EAAI,EACJJ,EAAW,SAAQ,CACrB,EAEA,OACA,UAAA,CAEEE,EAAYD,EAAa,IAC3B,CAAC,CACF,CAEL,CAAC,CACH,CCpFM,SAAUS,GAAqBC,EAAe,CAClD,OAAOC,EAAQ,SAACC,EAAQC,EAAU,CAChC,IAAIC,EAAW,GACfF,EAAO,UACLG,EACEF,EACA,SAACG,EAAK,CACJF,EAAW,GACXD,EAAW,KAAKG,CAAK,CACvB,EACA,UAAA,CACOF,GACHD,EAAW,KAAKH,CAAa,EAE/BG,EAAW,SAAQ,CACrB,CAAC,CACF,CAEL,CAAC,CACH,CCXM,SAAUI,GAAQC,EAAa,CACnC,OAAOA,GAAS,EAEZ,UAAA,CAAM,OAAAC,CAAA,EACNC,EAAQ,SAACC,EAAQC,EAAU,CACzB,IAAIC,EAAO,EACXF,EAAO,UACLG,EAAyBF,EAAY,SAACG,EAAK,CAIrC,EAAEF,GAAQL,IACZI,EAAW,KAAKG,CAAK,EAIjBP,GAASK,GACXD,EAAW,SAAQ,EAGzB,CAAC,CAAC,CAEN,CAAC,CACP,CC9BM,SAAUI,IAAc,CAC5B,OAAOC,EAAQ,SAACC,EAAQC,EAAU,CAChCD,EAAO,UAAUE,EAAyBD,EAAYE,EAAI,CAAC,CAC7D,CAAC,CACH,CCCM,SAAUC,GAASC,EAAQ,CAC/B,OAAOC,EAAI,UAAA,CAAM,OAAAD,CAAA,CAAK,CACxB,CC4CM,SAAUE,GACdC,EACAC,EAAmC,CAEnC,OAAIA,EAEK,SAACC,EAAqB,CAC3B,OAAAC,GAAOF,EAAkB,KAAKG,GAAK,CAAC,EAAGC,GAAc,CAAE,EAAGH,EAAO,KAAKH,GAAUC,CAAqB,CAAC,CAAC,CAAvG,EAGGM,GAAS,SAACC,EAAOC,EAAK,CAAK,OAAAC,EAAUT,EAAsBO,EAAOC,CAAK,CAAC,EAAE,KAAKJ,GAAK,CAAC,EAAGM,GAAMH,CAAK,CAAC,CAAzE,CAA0E,CAC9G,CCzCM,SAAUI,GAASC,EAAoBC,EAAyC,CAAzCA,IAAA,SAAAA,EAAAC,IAC3C,IAAMC,EAAWC,GAAMJ,EAAKC,CAAS,EACrC,OAAOI,GAAU,UAAA,CAAM,OAAAF,CAAA,CAAQ,CACjC,CC0EM,SAAUG,EACdC,EACAC,EAA0D,CAA1D,OAAAA,IAAA,SAAAA,EAA+BC,IAK/BF,EAAaA,GAAU,KAAVA,EAAcG,GAEpBC,EAAQ,SAACC,EAAQC,EAAU,CAGhC,IAAIC,EAEAC,EAAQ,GAEZH,EAAO,UACLI,EAAyBH,EAAY,SAACI,EAAK,CAEzC,IAAMC,EAAaV,EAAYS,CAAK,GAKhCF,GAAS,CAACR,EAAYO,EAAaI,CAAU,KAM/CH,EAAQ,GACRD,EAAcI,EAGdL,EAAW,KAAKI,CAAK,EAEzB,CAAC,CAAC,CAEN,CAAC,CACH,CAEA,SAASP,GAAeS,EAAQC,EAAM,CACpC,OAAOD,IAAMC,CACf,CCjHM,SAAUC,EAA8CC,EAAQC,EAAuC,CAC3G,OAAOC,EAAqB,SAACC,EAAMC,EAAI,CAAK,OAAAH,EAAUA,EAAQE,EAAEH,CAAG,EAAGI,EAAEJ,CAAG,CAAC,EAAIG,EAAEH,CAAG,IAAMI,EAAEJ,CAAG,CAApD,CAAqD,CACnG,CC7BM,SAAUK,GAAgBC,EAA6C,CAA7C,OAAAA,IAAA,SAAAA,EAAAC,IACvBC,EAAQ,SAACC,EAAQC,EAAU,CAChC,IAAIC,EAAW,GACfF,EAAO,UACLG,EACEF,EACA,SAACG,EAAK,CACJF,EAAW,GACXD,EAAW,KAAKG,CAAK,CACvB,EACA,UAAA,CAAM,OAACF,EAAWD,EAAW,SAAQ,EAAKA,EAAW,MAAMJ,EAAY,CAAE,CAAnE,CAAqE,CAC5E,CAEL,CAAC,CACH,CAEA,SAASC,IAAmB,CAC1B,OAAO,IAAIO,EACb,CCMM,SAAUC,IAAO,SAAIC,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,CAAA,EAAA,UAAAA,CAAA,EACzB,OAAO,SAACC,EAAqB,CAAK,OAAAC,GAAOD,EAAQE,EAAE,MAAA,OAAAC,EAAA,CAAA,EAAAC,EAAIN,CAAM,CAAA,CAAA,CAAA,CAA3B,CACpC,CCHM,SAAUO,EAAYC,EAAoB,CAC9C,OAAOC,EAAQ,SAACC,EAAQC,EAAU,CAGhC,GAAI,CACFD,EAAO,UAAUC,CAAU,UAE3BA,EAAW,IAAIH,CAAQ,EAE3B,CAAC,CACH,CCMM,SAAUI,GACdC,EACAC,EAAgB,CAEhB,IAAMC,EAAkB,UAAU,QAAU,EAC5C,OAAO,SAACC,EAAqB,CAC3B,OAAAA,EAAO,KACLH,EAAYI,EAAO,SAACC,EAAG,EAAC,CAAK,OAAAL,EAAUK,EAAG,EAAGF,CAAM,CAAtB,CAAuB,EAAIG,GACxDC,GAAK,CAAC,EACNL,EAAkBM,GAAeP,CAAa,EAAIQ,GAAa,UAAA,CAAM,OAAA,IAAIC,EAAJ,CAAgB,CAAC,CAHxF,CAKJ,CC/CM,SAAUC,GAAYC,EAAa,CACvC,OAAOA,GAAS,EACZ,UAAA,CAAM,OAAAC,CAAA,EACNC,EAAQ,SAACC,EAAQC,EAAU,CAKzB,IAAIC,EAAc,CAAA,EAClBF,EAAO,UACLG,EACEF,EACA,SAACG,EAAK,CAEJF,EAAO,KAAKE,CAAK,EAGjBP,EAAQK,EAAO,QAAUA,EAAO,MAAK,CACvC,EACA,UAAA,aAGE,QAAoBG,EAAAC,GAAAJ,CAAM,EAAAK,EAAAF,EAAA,KAAA,EAAA,CAAAE,EAAA,KAAAA,EAAAF,EAAA,KAAA,EAAE,CAAvB,IAAMD,EAAKG,EAAA,MACdN,EAAW,KAAKG,CAAK,oGAEvBH,EAAW,SAAQ,CACrB,EAEA,OACA,UAAA,CAEEC,EAAS,IACX,CAAC,CACF,CAEL,CAAC,CACP,CC1DM,SAAUM,IAAK,SAAIC,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,CAAA,EAAA,UAAAA,CAAA,EACvB,IAAMC,EAAYC,GAAaH,CAAI,EAC7BI,EAAaC,GAAUL,EAAM,GAAQ,EAC3C,OAAAA,EAAOM,GAAeN,CAAI,EAEnBO,EAAQ,SAACC,EAAQC,EAAU,CAChCC,GAASN,CAAU,EAAEO,GAAIC,EAAA,CAAEJ,CAAM,EAAAK,EAAMb,CAA6B,CAAA,EAAGE,CAAS,CAAC,EAAE,UAAUO,CAAU,CACzG,CAAC,CACH,CCcM,SAAUK,IAAS,SACvBC,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,CAAA,EAAA,UAAAA,CAAA,EAEA,OAAOC,GAAK,MAAA,OAAAC,EAAA,CAAA,EAAAC,EAAIJ,CAAY,CAAA,CAAA,CAC9B,CCmEM,SAAUK,GAAUC,EAAqC,OACzDC,EAAQ,IACRC,EAEJ,OAAIF,GAAiB,OACf,OAAOA,GAAkB,UACxBG,EAA4BH,EAAa,MAAzCC,EAAKE,IAAA,OAAG,IAAQA,EAAED,EAAUF,EAAa,OAE5CC,EAAQD,GAILC,GAAS,EACZ,UAAA,CAAM,OAAAG,CAAA,EACNC,EAAQ,SAACC,EAAQC,EAAU,CACzB,IAAIC,EAAQ,EACRC,EAEEC,EAAc,UAAA,CAGlB,GAFAD,GAAS,MAATA,EAAW,YAAW,EACtBA,EAAY,KACRP,GAAS,KAAM,CACjB,IAAMS,EAAW,OAAOT,GAAU,SAAWU,GAAMV,CAAK,EAAIW,EAAUX,EAAMM,CAAK,CAAC,EAC5EM,EAAqBC,EAAyBR,EAAY,UAAA,CAC9DO,EAAmB,YAAW,EAC9BE,EAAiB,CACnB,CAAC,EACDL,EAAS,UAAUG,CAAkB,OAErCE,EAAiB,CAErB,EAEMA,EAAoB,UAAA,CACxB,IAAIC,EAAY,GAChBR,EAAYH,EAAO,UACjBS,EAAyBR,EAAY,OAAW,UAAA,CAC1C,EAAEC,EAAQP,EACRQ,EACFC,EAAW,EAEXO,EAAY,GAGdV,EAAW,SAAQ,CAEvB,CAAC,CAAC,EAGAU,GACFP,EAAW,CAEf,EAEAM,EAAiB,CACnB,CAAC,CACP,CCpFM,SAAUE,GAAcC,EAA6DC,EAAQ,CAMjG,OAAOC,EAAQC,GAAcH,EAAaC,EAAW,UAAU,QAAU,EAAG,EAAI,CAAC,CACnF,CC+CM,SAAUG,GAASC,EAA4B,CAA5BA,IAAA,SAAAA,EAAA,CAAA,GACf,IAAAC,EAAgHD,EAAO,UAAvHE,EAASD,IAAA,OAAG,UAAA,CAAM,OAAA,IAAIE,CAAJ,EAAgBF,EAAEG,EAA4EJ,EAAO,aAAnFK,EAAYD,IAAA,OAAG,GAAIA,EAAEE,EAAuDN,EAAO,gBAA9DO,EAAeD,IAAA,OAAG,GAAIA,EAAEE,EAA+BR,EAAO,oBAAtCS,EAAmBD,IAAA,OAAG,GAAIA,EAUnH,OAAO,SAACE,EAAa,CACnB,IAAIC,EACAC,EACAC,EACAC,EAAW,EACXC,EAAe,GACfC,EAAa,GAEXC,EAAc,UAAA,CAClBL,GAAe,MAAfA,EAAiB,YAAW,EAC5BA,EAAkB,MACpB,EAGMM,GAAQ,UAAA,CACZD,EAAW,EACXN,EAAaE,EAAU,OACvBE,EAAeC,EAAa,EAC9B,EACMG,EAAsB,UAAA,CAG1B,IAAMC,EAAOT,EACbO,GAAK,EACLE,GAAI,MAAJA,EAAM,YAAW,CACnB,EAEA,OAAOC,EAAc,SAACC,EAAQC,GAAU,CACtCT,IACI,CAACE,GAAc,CAACD,GAClBE,EAAW,EAOb,IAAMO,GAAQX,EAAUA,GAAO,KAAPA,EAAWX,EAAS,EAO5CqB,GAAW,IAAI,UAAA,CACbT,IAKIA,IAAa,GAAK,CAACE,GAAc,CAACD,IACpCH,EAAkBa,GAAYN,EAAqBV,CAAmB,EAE1E,CAAC,EAIDe,GAAK,UAAUD,EAAU,EAGvB,CAACZ,GAIDG,EAAW,IAOXH,EAAa,IAAIe,GAAe,CAC9B,KAAM,SAACC,GAAK,CAAK,OAAAH,GAAK,KAAKG,EAAK,CAAf,EACjB,MAAO,SAACC,GAAG,CACTZ,EAAa,GACbC,EAAW,EACXL,EAAkBa,GAAYP,GAAOb,EAAcuB,EAAG,EACtDJ,GAAK,MAAMI,EAAG,CAChB,EACA,SAAU,UAAA,CACRb,EAAe,GACfE,EAAW,EACXL,EAAkBa,GAAYP,GAAOX,CAAe,EACpDiB,GAAK,SAAQ,CACf,EACD,EACDK,EAAUP,CAAM,EAAE,UAAUX,CAAU,EAE1C,CAAC,EAAED,CAAa,CAClB,CACF,CAEA,SAASe,GACPP,EACAY,EAAoD,SACpDC,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,EAAA,CAAA,EAAA,UAAAA,CAAA,EAEA,GAAIF,IAAO,GAAM,CACfZ,EAAK,EACL,OAGF,GAAIY,IAAO,GAIX,KAAMG,EAAe,IAAIP,GAAe,CACtC,KAAM,UAAA,CACJO,EAAa,YAAW,EACxBf,EAAK,CACP,EACD,EAED,OAAOW,EAAUC,EAAE,MAAA,OAAAI,EAAA,CAAA,EAAAC,EAAIJ,CAAI,CAAA,CAAA,CAAA,EAAG,UAAUE,CAAY,EACtD,CChHM,SAAUG,EACdC,EACAC,EACAC,EAAyB,WAErBC,EACAC,EAAW,GACf,OAAIJ,GAAsB,OAAOA,GAAuB,UACnDK,EAA8EL,EAAkB,WAAhGG,EAAUE,IAAA,OAAG,IAAQA,EAAEC,EAAuDN,EAAkB,WAAzEC,EAAUK,IAAA,OAAG,IAAQA,EAAEC,EAAgCP,EAAkB,SAAlDI,EAAQG,IAAA,OAAG,GAAKA,EAAEL,EAAcF,EAAkB,WAEnGG,EAAcH,GAAkB,KAAlBA,EAAsB,IAE/BQ,GAAS,CACd,UAAW,UAAA,CAAM,OAAA,IAAIC,GAAcN,EAAYF,EAAYC,CAAS,CAAnD,EACjB,aAAc,GACd,gBAAiB,GACjB,oBAAqBE,EACtB,CACH,CCxIM,SAAUM,GAAQC,EAAa,CACnC,OAAOC,EAAO,SAACC,EAAGC,EAAK,CAAK,OAAAH,GAASG,CAAT,CAAc,CAC5C,CCaM,SAAUC,GAAaC,EAA8B,CACzD,OAAOC,EAAQ,SAACC,EAAQC,EAAU,CAChC,IAAIC,EAAS,GAEPC,EAAiBC,EACrBH,EACA,UAAA,CACEE,GAAc,MAAdA,EAAgB,YAAW,EAC3BD,EAAS,EACX,EACAG,EAAI,EAGNC,EAAUR,CAAQ,EAAE,UAAUK,CAAc,EAE5CH,EAAO,UAAUI,EAAyBH,EAAY,SAACM,EAAK,CAAK,OAAAL,GAAUD,EAAW,KAAKM,CAAK,CAA/B,CAAgC,CAAC,CACpG,CAAC,CACH,CCVM,SAAUC,GAAS,SAAOC,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,CAAA,EAAA,UAAAA,CAAA,EAC9B,IAAMC,EAAYC,GAAaH,CAAM,EACrC,OAAOI,EAAQ,SAACC,EAAQC,EAAU,EAI/BJ,EAAYK,GAAOP,EAAQK,EAAQH,CAAS,EAAIK,GAAOP,EAAQK,CAAM,GAAG,UAAUC,CAAU,CAC/F,CAAC,CACH,CCmBM,SAAUE,EACdC,EACAC,EAA6G,CAE7G,OAAOC,EAAQ,SAACC,EAAQC,EAAU,CAChC,IAAIC,EAAyD,KACzDC,EAAQ,EAERC,EAAa,GAIXC,EAAgB,UAAA,CAAM,OAAAD,GAAc,CAACF,GAAmBD,EAAW,SAAQ,CAArD,EAE5BD,EAAO,UACLM,EACEL,EACA,SAACM,EAAK,CAEJL,GAAe,MAAfA,EAAiB,YAAW,EAC5B,IAAIM,EAAa,EACXC,EAAaN,IAEnBO,EAAUb,EAAQU,EAAOE,CAAU,CAAC,EAAE,UACnCP,EAAkBI,EACjBL,EAIA,SAACU,EAAU,CAAK,OAAAV,EAAW,KAAKH,EAAiBA,EAAeS,EAAOI,EAAYF,EAAYD,GAAY,EAAIG,CAAU,CAAzG,EAChB,UAAA,CAIET,EAAkB,KAClBG,EAAa,CACf,CAAC,CACD,CAEN,EACA,UAAA,CACED,EAAa,GACbC,EAAa,CACf,CAAC,CACF,CAEL,CAAC,CACH,CCvFM,SAAUO,EAAaC,EAA8B,CACzD,OAAOC,EAAQ,SAACC,EAAQC,EAAU,CAChCC,EAAUJ,CAAQ,EAAE,UAAUK,EAAyBF,EAAY,UAAA,CAAM,OAAAA,EAAW,SAAQ,CAAnB,EAAuBG,EAAI,CAAC,EACrG,CAACH,EAAW,QAAUD,EAAO,UAAUC,CAAU,CACnD,CAAC,CACH,CCIM,SAAUI,GAAaC,EAAiDC,EAAiB,CAAjB,OAAAA,IAAA,SAAAA,EAAA,IACrEC,EAAQ,SAACC,EAAQC,EAAU,CAChC,IAAIC,EAAQ,EACZF,EAAO,UACLG,EAAyBF,EAAY,SAACG,EAAK,CACzC,IAAMC,EAASR,EAAUO,EAAOF,GAAO,GACtCG,GAAUP,IAAcG,EAAW,KAAKG,CAAK,EAC9C,CAACC,GAAUJ,EAAW,SAAQ,CAChC,CAAC,CAAC,CAEN,CAAC,CACH,CCqGM,SAAUK,EACdC,EACAC,EACAC,EAA8B,CAK9B,IAAMC,EACJC,EAAWJ,CAAc,GAAKC,GAASC,EAElC,CAAE,KAAMF,EAA2E,MAAKC,EAAE,SAAQC,CAAA,EACnGF,EAEN,OAAOG,EACHE,EAAQ,SAACC,EAAQC,EAAU,QACzBC,EAAAL,EAAY,aAAS,MAAAK,IAAA,QAAAA,EAAA,KAArBL,CAAW,EACX,IAAIM,EAAU,GACdH,EAAO,UACLI,EACEH,EACA,SAACI,EAAK,QACJH,EAAAL,EAAY,QAAI,MAAAK,IAAA,QAAAA,EAAA,KAAhBL,EAAmBQ,CAAK,EACxBJ,EAAW,KAAKI,CAAK,CACvB,EACA,UAAA,OACEF,EAAU,IACVD,EAAAL,EAAY,YAAQ,MAAAK,IAAA,QAAAA,EAAA,KAApBL,CAAW,EACXI,EAAW,SAAQ,CACrB,EACA,SAACK,EAAG,OACFH,EAAU,IACVD,EAAAL,EAAY,SAAK,MAAAK,IAAA,QAAAA,EAAA,KAAjBL,EAAoBS,CAAG,EACvBL,EAAW,MAAMK,CAAG,CACtB,EACA,UAAA,SACMH,KACFD,EAAAL,EAAY,eAAW,MAAAK,IAAA,QAAAA,EAAA,KAAvBL,CAAW,IAEbU,EAAAV,EAAY,YAAQ,MAAAU,IAAA,QAAAA,EAAA,KAApBV,CAAW,CACb,CAAC,CACF,CAEL,CAAC,EAIDW,EACN,CCnIM,SAAUC,GAAYC,EAAsDC,EAAuB,CACvG,OAAOC,EAAQ,SAACC,EAAQC,EAAU,CAC1B,IAAAC,EAAuCJ,GAAM,KAANA,EAAU,CAAA,EAA/CK,EAAAD,EAAA,QAAAE,EAAOD,IAAA,OAAG,GAAIA,EAAEE,EAAAH,EAAA,SAAAI,EAAQD,IAAA,OAAG,GAAKA,EACpCE,EAAW,GACXC,EAAsB,KACtBC,EAAiC,KACjCC,EAAa,GAEXC,EAAgB,UAAA,CACpBF,GAAS,MAATA,EAAW,YAAW,EACtBA,EAAY,KACRH,IACFM,EAAI,EACJF,GAAcT,EAAW,SAAQ,EAErC,EAEMY,EAAoB,UAAA,CACxBJ,EAAY,KACZC,GAAcT,EAAW,SAAQ,CACnC,EAEMa,EAAgB,SAACC,GAAQ,CAC7B,OAACN,EAAYO,EAAUnB,EAAiBkB,EAAK,CAAC,EAAE,UAAUE,EAAyBhB,EAAYU,EAAeE,CAAiB,CAAC,CAAhI,EAEID,EAAO,UAAA,CACX,GAAIL,EAAU,CAIZA,EAAW,GACX,IAAMQ,GAAQP,EACdA,EAAY,KAEZP,EAAW,KAAKc,EAAK,EACrB,CAACL,GAAcI,EAAcC,EAAK,EAEtC,EAEAf,EAAO,UACLiB,EACEhB,EAMA,SAACc,GAAK,CACJR,EAAW,GACXC,EAAYO,GACZ,EAAEN,GAAa,CAACA,EAAU,UAAYL,EAAUQ,EAAI,EAAKE,EAAcC,EAAK,EAC9E,EACA,UAAA,CACEL,EAAa,GACb,EAAEJ,GAAYC,GAAYE,GAAa,CAACA,EAAU,SAAWR,EAAW,SAAQ,CAClF,CAAC,CACF,CAEL,CAAC,CACH,CCxFM,SAAUiB,GACdC,EACAC,EACAC,EAAuB,CADvBD,IAAA,SAAAA,EAAAE,IAGA,IAAMC,EAAYC,GAAML,EAAUC,CAAS,EAC3C,OAAOK,GAAS,UAAA,CAAM,OAAAF,CAAA,EAAWF,CAAM,CACzC,CCJM,SAAUK,IAAc,SAAOC,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,CAAA,EAAA,UAAAA,CAAA,EACnC,IAAMC,EAAUC,GAAkBH,CAAM,EAExC,OAAOI,EAAQ,SAACC,EAAQC,EAAU,CAehC,QAdMC,EAAMP,EAAO,OACbQ,EAAc,IAAI,MAAMD,CAAG,EAI7BE,EAAWT,EAAO,IAAI,UAAA,CAAM,MAAA,EAAA,CAAK,EAGjCU,EAAQ,cAMHC,EAAC,CACRC,EAAUZ,EAAOW,CAAC,CAAC,EAAE,UACnBE,EACEP,EACA,SAACQ,EAAK,CACJN,EAAYG,CAAC,EAAIG,EACb,CAACJ,GAAS,CAACD,EAASE,CAAC,IAEvBF,EAASE,CAAC,EAAI,IAKbD,EAAQD,EAAS,MAAMM,EAAQ,KAAON,EAAW,MAEtD,EAGAO,EAAI,CACL,GAnBIL,EAAI,EAAGA,EAAIJ,EAAKI,MAAhBA,CAAC,EAwBVN,EAAO,UACLQ,EAAyBP,EAAY,SAACQ,EAAK,CACzC,GAAIJ,EAAO,CAET,IAAMO,EAAMC,EAAA,CAAIJ,CAAK,EAAAK,EAAKX,CAAW,CAAA,EACrCF,EAAW,KAAKJ,EAAUA,EAAO,MAAA,OAAAgB,EAAA,CAAA,EAAAC,EAAIF,CAAM,CAAA,CAAA,EAAIA,CAAM,EAEzD,CAAC,CAAC,CAEN,CAAC,CACH,CCxFM,SAAUG,IAAG,SAAOC,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,CAAA,EAAA,UAAAA,CAAA,EACxB,OAAOC,EAAQ,SAACC,EAAQC,EAAU,CAChCL,GAAS,MAAA,OAAAM,EAAA,CAACF,CAA8B,EAAAG,EAAMN,CAAuC,CAAA,CAAA,EAAE,UAAUI,CAAU,CAC7G,CAAC,CACH,CCCM,SAAUG,IAAO,SAAkCC,EAAA,CAAA,EAAAC,EAAA,EAAAA,EAAA,UAAA,OAAAA,IAAAD,EAAAC,CAAA,EAAA,UAAAA,CAAA,EACvD,OAAOC,GAAG,MAAA,OAAAC,EAAA,CAAA,EAAAC,EAAIJ,CAAW,CAAA,CAAA,CAC3B,CCYO,SAASK,IAAmC,CACjD,IAAMC,EAAY,IAAIC,GAAwB,CAAC,EAC/C,OAAAC,EAAU,SAAU,mBAAoB,CAAE,KAAM,EAAK,CAAC,EACnD,UAAU,IAAMF,EAAU,KAAK,QAAQ,CAAC,EAGpCA,CACT,CCHO,SAASG,EACdC,EAAkBC,EAAmB,SAChC,CACL,OAAO,MAAM,KAAKA,EAAK,iBAAoBD,CAAQ,CAAC,CACtD,CAuBO,SAASE,EACdF,EAAkBC,EAAmB,SAClC,CACH,IAAME,EAAKC,GAAsBJ,EAAUC,CAAI,EAC/C,GAAI,OAAOE,GAAO,YAChB,MAAM,IAAI,eACR,8BAA8BH,CAAQ,iBACxC,EAGF,OAAOG,CACT,CAsBO,SAASC,GACdJ,EAAkBC,EAAmB,SACtB,CACf,OAAOA,EAAK,cAAiBD,CAAQ,GAAK,MAC5C,CAOO,SAASK,IAA4C,CAnH5D,IAAAC,EAAAC,EAAAC,EAAAC,EAoHE,OACEA,GAAAD,GAAAD,GAAAD,EAAA,SAAS,gBAAT,YAAAA,EAAwB,aAAxB,YAAAC,EAAoC,gBAApC,KAAAC,EACA,SAAS,gBADT,KAAAC,EAEA,MAEJ,CCvEA,IAAMC,GAAYC,EAChBC,EAAU,SAAS,KAAM,SAAS,EAClCA,EAAU,SAAS,KAAM,UAAU,CACrC,EACG,KACCC,GAAa,CAAC,EACdC,EAAU,MAAS,EACnBC,EAAI,IAAMC,GAAiB,GAAK,SAAS,IAAI,EAC7CC,EAAY,CAAC,CACf,EAaK,SAASC,GACdC,EACqB,CACrB,OAAOT,GACJ,KACCK,EAAIK,GAAUD,EAAG,SAASC,CAAM,CAAC,EACjCC,EAAqB,CACvB,CACJ,CCpCO,SAASC,GACdC,EAAiBC,EACI,CACrB,OAAOC,EACLC,EAAUH,EAAI,YAAY,EAAE,KAAKI,EAAI,IAAM,EAAI,CAAC,EAChDD,EAAUH,EAAI,YAAY,EAAE,KAAKI,EAAI,IAAM,EAAK,CAAC,CACnD,EACG,KACCH,EAAWI,GAAaJ,CAAQ,EAAIK,GACpCC,EAAU,EAAK,CACjB,CACJ,CCAO,SAASC,GACdC,EACe,CACf,MAAO,CACL,EAAGA,EAAG,WACN,EAAGA,EAAG,SACR,CACF,CAWO,SAASC,GACdD,EAC2B,CAC3B,OAAOE,EACLC,EAAU,OAAQ,MAAM,EACxBA,EAAU,OAAQ,QAAQ,CAC5B,EACG,KACCC,GAAU,EAAGC,EAAuB,EACpCC,EAAI,IAAMP,GAAiBC,CAAE,CAAC,EAC9BO,EAAUR,GAAiBC,CAAE,CAAC,CAChC,CACJ,CCxCO,SAASQ,GACdC,EACe,CACf,MAAO,CACL,EAAGA,EAAG,WACN,EAAGA,EAAG,SACR,CACF,CAWO,SAASC,GACdD,EAC2B,CAC3B,OAAOE,EACLC,EAAUH,EAAI,QAAQ,EACtBG,EAAU,OAAQ,QAAQ,CAC5B,EACG,KACCC,GAAU,EAAGC,EAAuB,EACpCC,EAAI,IAAMP,GAAwBC,CAAE,CAAC,EACrCO,EAAUR,GAAwBC,CAAE,CAAC,CACvC,CACJ,CCnBA,SAASQ,GAAYC,EAAiBC,EAA8B,CAGlE,GAAI,OAAOA,GAAU,UAAY,OAAOA,GAAU,SAChDD,EAAG,WAAaC,EAAM,SAAS,UAGtBA,aAAiB,KAC1BD,EAAG,YAAYC,CAAK,UAGX,MAAM,QAAQA,CAAK,EAC5B,QAAWC,KAAQD,EACjBF,GAAYC,EAAIE,CAAI,CAE1B,CAyBO,SAASC,EACdC,EAAaC,KAAmCC,EAC7C,CACH,IAAMN,EAAK,SAAS,cAAcI,CAAG,EAGrC,GAAIC,EACF,QAAWE,KAAQ,OAAO,KAAKF,CAAU,EACnC,OAAOA,EAAWE,CAAI,GAAM,cAI5B,OAAOF,EAAWE,CAAI,GAAM,UAC9BP,EAAG,aAAaO,EAAMF,EAAWE,CAAI,CAAC,EAEtCP,EAAG,aAAaO,EAAM,EAAE,GAI9B,QAAWN,KAASK,EAClBP,GAAYC,EAAIC,CAAK,EAGvB,OAAOD,CACT,CC9EO,SAASQ,GAAMC,EAAuB,CAC3C,GAAIA,EAAQ,IAAK,CACf,IAAMC,EAAS,GAAGD,EAAQ,KAAO,IAAO,IACxC,MAAO,KAAKA,EAAQ,MAAY,KAAM,QAAQC,CAAM,CAAC,GACvD,KACE,QAAOD,EAAM,SAAS,CAE1B,CCCO,SAASE,GAAYC,EAA+B,CACzD,IAAMC,EAASC,EAAE,SAAU,CAAE,IAAAF,CAAI,CAAC,EAClC,OAAOG,EAAM,KACX,SAAS,KAAK,YAAYF,CAAM,EACzBG,EACLC,EAAUJ,EAAQ,MAAM,EACxBI,EAAUJ,EAAQ,OAAO,EACtB,KACCK,EAAU,IACRC,GAAW,IAAM,IAAI,eAAe,mBAAmBP,CAAG,EAAE,CAAC,CAC9D,CACH,CACJ,EACG,KACCQ,EAAI,IAAG,EAAY,EACnBC,EAAS,IAAM,SAAS,KAAK,YAAYR,CAAM,CAAC,EAChDS,GAAK,CAAC,CACR,EACH,CACH,CCVA,IAAMC,GAAS,IAAIC,EAiBbC,GAAYC,EAAM,IACtB,OAAO,gBAAmB,YACtBC,GAAY,4CAA4C,EACxDC,EAAG,MAAS,CACjB,EACE,KACCC,EAAI,IAAM,IAAI,eAAeC,GAAW,CACtC,QAAWC,KAASD,EAClBP,GAAO,KAAKQ,CAAK,CACrB,CAAC,CAAC,EACFC,EAAUC,GAAYC,EAAMC,GAAOP,EAAGK,CAAQ,CAAC,EAC5C,KACCG,EAAS,IAAMH,EAAS,WAAW,CAAC,CACtC,CACF,EACAI,EAAY,CAAC,CACf,EAaK,SAASC,GACdC,EACa,CACb,MAAO,CACL,MAAQA,EAAG,YACX,OAAQA,EAAG,YACb,CACF,CAuBO,SAASC,GACdD,EACyB,CACzB,OAAOd,GACJ,KACCgB,EAAIR,GAAYA,EAAS,QAAQM,CAAE,CAAC,EACpCP,EAAUC,GAAYV,GACnB,KACCmB,EAAO,CAAC,CAAE,OAAAC,CAAO,IAAMA,IAAWJ,CAAE,EACpCH,EAAS,IAAMH,EAAS,UAAUM,CAAE,CAAC,EACrCV,EAAI,IAAMS,GAAeC,CAAE,CAAC,CAC9B,CACF,EACAK,EAAUN,GAAeC,CAAE,CAAC,CAC9B,CACJ,CCnHO,SAASM,GACdC,EACa,CACb,MAAO,CACL,MAAQA,EAAG,YACX,OAAQA,EAAG,YACb,CACF,CASO,SAASC,GACdD,EACyB,CACzB,IAAIE,EAASF,EAAG,cAChB,KAAOE,IAEHF,EAAG,aAAgBE,EAAO,aAC1BF,EAAG,cAAgBE,EAAO,eAE1BA,GAAUF,EAAKE,GAAQ,cAK3B,OAAOA,EAASF,EAAK,MACvB,CCfA,IAAMG,GAAS,IAAIC,EAUbC,GAAYC,EAAM,IAAMC,EAC5B,IAAI,qBAAqBC,GAAW,CAClC,QAAWC,KAASD,EAClBL,GAAO,KAAKM,CAAK,CACrB,EAAG,CACD,UAAW,CACb,CAAC,CACH,CAAC,EACE,KACCC,EAAUC,GAAYC,EAAMC,GAAON,EAAGI,CAAQ,CAAC,EAC5C,KACCG,EAAS,IAAMH,EAAS,WAAW,CAAC,CACtC,CACF,EACAI,EAAY,CAAC,CACf,EAaK,SAASC,GACdC,EACqB,CACrB,OAAOZ,GACJ,KACCa,EAAIP,GAAYA,EAAS,QAAQM,CAAE,CAAC,EACpCP,EAAUC,GAAYR,GACnB,KACCgB,EAAO,CAAC,CAAE,OAAAC,CAAO,IAAMA,IAAWH,CAAE,EACpCH,EAAS,IAAMH,EAAS,UAAUM,CAAE,CAAC,EACrCI,EAAI,CAAC,CAAE,eAAAC,CAAe,IAAMA,CAAc,CAC5C,CACF,CACF,CACJ,CAaO,SAASC,GACdN,EAAiBO,EAAY,GACR,CACrB,OAAOC,GAA0BR,CAAE,EAChC,KACCI,EAAI,CAAC,CAAE,EAAAK,CAAE,IAAM,CACb,IAAMC,EAAUC,GAAeX,CAAE,EAC3BY,EAAUC,GAAsBb,CAAE,EACxC,OAAOS,GACLG,EAAQ,OAASF,EAAQ,OAASH,CAEtC,CAAC,EACDO,EAAqB,CACvB,CACJ,CCjFA,IAAMC,GAA4C,CAChD,OAAQC,EAAW,yBAAyB,EAC5C,OAAQA,EAAW,yBAAyB,CAC9C,EAaO,SAASC,GAAUC,EAAuB,CAC/C,OAAOH,GAAQG,CAAI,EAAE,OACvB,CAaO,SAASC,GAAUD,EAAcE,EAAsB,CACxDL,GAAQG,CAAI,EAAE,UAAYE,GAC5BL,GAAQG,CAAI,EAAE,MAAM,CACxB,CAWO,SAASG,GAAYH,EAAmC,CAC7D,IAAMI,EAAKP,GAAQG,CAAI,EACvB,OAAOK,EAAUD,EAAI,QAAQ,EAC1B,KACCE,EAAI,IAAMF,EAAG,OAAO,EACpBG,EAAUH,EAAG,OAAO,CACtB,CACJ,CC9BA,SAASI,GACPC,EAAiBC,EACR,CACT,OAAQD,EAAG,YAAa,CAGtB,KAAK,iBAEH,OAAIA,EAAG,OAAS,QACP,SAAS,KAAKC,CAAI,EAElB,GAGX,KAAK,kBACL,KAAK,oBACH,MAAO,GAGT,QACE,OAAOD,EAAG,iBACd,CACF,CAWO,SAASE,IAAwC,CACtD,OAAOC,EACLC,EAAU,OAAQ,kBAAkB,EAAE,KAAKC,EAAI,IAAM,EAAI,CAAC,EAC1DD,EAAU,OAAQ,gBAAgB,EAAE,KAAKC,EAAI,IAAM,EAAK,CAAC,CAC3D,EACG,KACCC,EAAU,EAAK,CACjB,CACJ,CAOO,SAASC,IAAsC,CACpD,IAAMC,EAAYJ,EAAyB,OAAQ,SAAS,EACzD,KACCK,EAAOC,GAAM,EAAEA,EAAG,SAAWA,EAAG,QAAQ,EACxCL,EAAIK,IAAO,CACT,KAAMC,GAAU,QAAQ,EAAI,SAAW,SACvC,KAAMD,EAAG,IACT,OAAQ,CACNA,EAAG,eAAe,EAClBA,EAAG,gBAAgB,CACrB,CACF,EAAc,EACdD,EAAO,CAAC,CAAE,KAAAG,EAAM,KAAAX,CAAK,IAAM,CACzB,GAAIW,IAAS,SAAU,CACrB,IAAMC,EAASC,GAAiB,EAChC,GAAI,OAAOD,GAAW,YACpB,MAAO,CAACd,GAAwBc,EAAQZ,CAAI,CAChD,CACA,MAAO,EACT,CAAC,EACDc,GAAM,CACR,EAGF,OAAOb,GAAiB,EACrB,KACCc,EAAUH,GAAWA,EAAqBI,EAAZT,CAAiB,CACjD,CACJ,CC1GO,SAASU,IAAmB,CACjC,OAAO,IAAI,IAAI,SAAS,IAAI,CAC9B,CAgBO,SAASC,GACdC,EAA4BC,EAAW,GACjC,CACN,GAAIC,EAAQ,oBAAoB,GAAK,CAACD,EAAU,CAC9C,IAAME,EAAKC,EAAE,IAAK,CAAE,KAAMJ,EAAI,IAAK,CAAC,EACpC,SAAS,KAAK,YAAYG,CAAE,EAC5BA,EAAG,MAAM,EACTA,EAAG,OAAO,CAIZ,MACE,SAAS,KAAOH,EAAI,IAExB,CASO,SAASK,IAA8B,CAC5C,OAAO,IAAIC,CACb,CCxCO,SAASC,IAA0B,CACxC,OAAO,SAAS,KAAK,MAAM,CAAC,CAC9B,CAYO,SAASC,GAAgBC,EAAoB,CAClD,IAAMC,EAAKC,EAAE,IAAK,CAAE,KAAMF,CAAK,CAAC,EAChCC,EAAG,iBAAiB,QAASE,GAAMA,EAAG,gBAAgB,CAAC,EACvDF,EAAG,MAAM,CACX,CAWO,SAASG,GACdC,EACoB,CACpB,OAAOC,EACLC,EAA2B,OAAQ,YAAY,EAC/CF,CACF,EACG,KACCG,EAAIV,EAAe,EACnBW,EAAUX,GAAgB,CAAC,EAC3BY,EAAOV,GAAQA,EAAK,OAAS,CAAC,EAC9BW,EAAY,CAAC,CACf,CACJ,CASO,SAASC,GACdP,EACyB,CACzB,OAAOD,GAAkBC,CAAS,EAC/B,KACCG,EAAIK,GAAMC,GAAmB,QAAQD,CAAE,IAAI,CAAE,EAC7CH,EAAOT,GAAM,OAAOA,GAAO,WAAW,CACxC,CACJ,CCtDO,SAASc,GAAWC,EAAoC,CAC7D,IAAMC,EAAQ,WAAWD,CAAK,EAC9B,OAAOE,GAA0BC,GAC/BF,EAAM,YAAY,IAAME,EAAKF,EAAM,OAAO,CAAC,CAC5C,EACE,KACCG,EAAUH,EAAM,OAAO,CACzB,CACJ,CAOO,SAASI,IAAkC,CAChD,IAAMJ,EAAQ,WAAW,OAAO,EAChC,OAAOK,EACLC,EAAU,OAAQ,aAAa,EAAE,KAAKC,EAAI,IAAM,EAAI,CAAC,EACrDD,EAAU,OAAQ,YAAY,EAAE,KAAKC,EAAI,IAAM,EAAK,CAAC,CACvD,EACG,KACCJ,EAAUH,EAAM,OAAO,CACzB,CACJ,CAcO,SAASQ,GACdC,EAA6BC,EACd,CACf,OAAOD,EACJ,KACCE,EAAUC,GAAUA,EAASF,EAAQ,EAAIG,CAAK,CAChD,CACJ,CC/BO,SAASC,GACdC,EAAmBC,EACD,CAClB,OAAO,IAAIC,EAAiBC,GAAY,CACtC,IAAMC,EAAM,IAAI,eAChB,OAAAA,EAAI,KAAK,MAAO,GAAGJ,CAAG,EAAE,EACxBI,EAAI,aAAe,OAGnBA,EAAI,iBAAiB,OAAQ,IAAM,CAC7BA,EAAI,QAAU,KAAOA,EAAI,OAAS,KACpCD,EAAS,KAAKC,EAAI,QAAQ,EAC1BD,EAAS,SAAS,GAIlBA,EAAS,MAAM,IAAI,MAAMC,EAAI,UAAU,CAAC,CAE5C,CAAC,EAGDA,EAAI,iBAAiB,QAAS,IAAM,CAClCD,EAAS,MAAM,IAAI,MAAM,eAAe,CAAC,CAC3C,CAAC,EAGDC,EAAI,iBAAiB,QAAS,IAAM,CAClCD,EAAS,SAAS,CACpB,CAAC,EAGG,OAAOF,GAAA,YAAAA,EAAS,YAAc,cAChCG,EAAI,iBAAiB,WAAYC,GAAS,CA/FhD,IAAAC,EAgGQ,GAAID,EAAM,iBACRJ,EAAQ,UAAW,KAAMI,EAAM,OAASA,EAAM,MAAS,GAAG,MAIrD,CACL,IAAME,GAASD,EAAAF,EAAI,kBAAkB,gBAAgB,IAAtC,KAAAE,EAA2C,EAC1DL,EAAQ,UAAW,KAAMI,EAAM,OAAS,CAACE,EAAU,GAAG,CACxD,CACF,CAAC,EAGDN,EAAQ,UAAU,KAAK,CAAC,GAI1BG,EAAI,KAAK,EACF,IAAMA,EAAI,MAAM,CACzB,CAAC,CACH,CAcO,SAASI,GACdR,EAAmBC,EACJ,CACf,OAAOF,GAAQC,EAAKC,CAAO,EACxB,KACCQ,EAAUC,GAAOA,EAAI,KAAK,CAAC,EAC3BC,EAAIC,GAAQ,KAAK,MAAMA,CAAI,CAAM,EACjCC,EAAY,CAAC,CACf,CACJ,CAUO,SAASC,GACdd,EAAmBC,EACG,CACtB,IAAMc,EAAM,IAAI,UAChB,OAAOhB,GAAQC,EAAKC,CAAO,EACxB,KACCQ,EAAUC,GAAOA,EAAI,KAAK,CAAC,EAC3BC,EAAID,GAAOK,EAAI,gBAAgBL,EAAK,WAAW,CAAC,EAChDG,EAAY,CAAC,CACf,CACJ,CAUO,SAASG,GACdhB,EAAmBC,EACG,CACtB,IAAMc,EAAM,IAAI,UAChB,OAAOhB,GAAQC,EAAKC,CAAO,EACxB,KACCQ,EAAUC,GAAOA,EAAI,KAAK,CAAC,EAC3BC,EAAID,GAAOK,EAAI,gBAAgBL,EAAK,UAAU,CAAC,EAC/CG,EAAY,CAAC,CACf,CACJ,CC5HO,SAASI,IAAoC,CAClD,MAAO,CACL,EAAG,KAAK,IAAI,EAAG,OAAO,EACtB,EAAG,KAAK,IAAI,EAAG,OAAO,CACxB,CACF,CASO,SAASC,IAAkD,CAChE,OAAOC,EACLC,EAAU,OAAQ,SAAU,CAAE,QAAS,EAAK,CAAC,EAC7CA,EAAU,OAAQ,SAAU,CAAE,QAAS,EAAK,CAAC,CAC/C,EACG,KACCC,EAAIJ,EAAiB,EACrBK,EAAUL,GAAkB,CAAC,CAC/B,CACJ,CC3BO,SAASM,IAAgC,CAC9C,MAAO,CACL,MAAQ,WACR,OAAQ,WACV,CACF,CASO,SAASC,IAA8C,CAC5D,OAAOC,EAAU,OAAQ,SAAU,CAAE,QAAS,EAAK,CAAC,EACjD,KACCC,EAAIH,EAAe,EACnBI,EAAUJ,GAAgB,CAAC,CAC7B,CACJ,CCXO,SAASK,IAAsC,CACpD,OAAOC,EAAc,CACnBC,GAAoB,EACpBC,GAAkB,CACpB,CAAC,EACE,KACCC,EAAI,CAAC,CAACC,EAAQC,CAAI,KAAO,CAAE,OAAAD,EAAQ,KAAAC,CAAK,EAAE,EAC1CC,EAAY,CAAC,CACf,CACJ,CCVO,SAASC,GACdC,EAAiB,CAAE,UAAAC,EAAW,QAAAC,CAAQ,EAChB,CACtB,IAAMC,EAAQF,EACX,KACCG,EAAwB,MAAM,CAChC,EAGIC,EAAUC,EAAc,CAACH,EAAOD,CAAO,CAAC,EAC3C,KACCK,EAAI,IAAMC,GAAiBR,CAAE,CAAC,CAChC,EAGF,OAAOM,EAAc,CAACJ,EAASD,EAAWI,CAAO,CAAC,EAC/C,KACCE,EAAI,CAAC,CAAC,CAAE,OAAAE,CAAO,EAAG,CAAE,OAAAC,EAAQ,KAAAC,CAAK,EAAG,CAAE,EAAAC,EAAG,EAAAC,CAAE,CAAC,KAAO,CACjD,OAAQ,CACN,EAAGH,EAAO,EAAIE,EACd,EAAGF,EAAO,EAAIG,EAAIJ,CACpB,EACA,KAAAE,CACF,EAAE,CACJ,CACJ,CCzBA,SAASG,GAAQC,EAA+B,CAC9C,OAAOC,EAA8BD,EAAQ,UAAWE,GAAMA,EAAG,IAAI,CACvE,CAWA,SAASC,GAAQH,EAA4B,CAC3C,IAAMI,EAAQ,IAAIC,EAClB,OAAAD,EAAM,UAAUE,GAAQN,EAAO,YAAYM,CAAI,CAAC,EAGzCF,CACT,CAgBO,SAASG,GACdC,EAAaR,EAAS,IAAI,OAAOQ,CAAG,EACxB,CACZ,IAAMC,EAAQV,GAAQC,CAAM,EACtBI,EAAQD,GAAQH,CAAM,EAGtBU,EAAU,IAAIL,EACpBK,EAAQ,UAAUN,CAAK,EAGvB,IAAMO,EAAQP,EAAM,KAAKQ,GAAe,EAAGC,GAAQ,EAAI,CAAC,EACxD,OAAOH,EACJ,KACCE,GAAe,EACfE,GAAUL,EAAM,KAAKM,EAAUJ,CAAK,CAAC,CAAC,EACtCK,GAAM,CACR,CACJ,CCLA,IAAMC,GAASC,EAAW,WAAW,EAC/BC,GAAiB,KAAK,MAAMF,GAAO,WAAY,EACrDE,GAAO,KAAO,GAAG,IAAI,IAAIA,GAAO,KAAMC,GAAY,CAAC,CAAC,GAW7C,SAASC,IAAwB,CACtC,OAAOF,EACT,CASO,SAASG,EAAQC,EAAqB,CAC3C,OAAOJ,GAAO,SAAS,SAASI,CAAI,CACtC,CAUO,SAASC,GACdC,EAAkBC,EACV,CACR,OAAO,OAAOA,GAAU,YACpBP,GAAO,aAAaM,CAAG,EAAE,QAAQ,IAAKC,EAAM,SAAS,CAAC,EACtDP,GAAO,aAAaM,CAAG,CAC7B,CC/BO,SAASE,GACdC,EAASC,EAAmB,SACP,CACrB,OAAOC,EAAW,sBAAsBF,CAAI,IAAKC,CAAI,CACvD,CAYO,SAASE,GACdH,EAASC,EAAmB,SACL,CACvB,OAAOG,EAAY,sBAAsBJ,CAAI,IAAKC,CAAI,CACxD,CC7EO,SAASI,GACdC,EACsB,CACtB,IAAMC,EAASC,EAAW,6BAA8BF,CAAE,EAC1D,OAAOG,EAAUF,EAAQ,QAAS,CAAE,KAAM,EAAK,CAAC,EAC7C,KACCG,EAAI,IAAMF,EAAW,cAAeF,CAAE,CAAC,EACvCI,EAAIC,IAAY,CAAE,KAAM,UAAUA,EAAQ,SAAS,CAAE,EAAE,CACzD,CACJ,CASO,SAASC,GACdN,EACiC,CACjC,GAAI,CAACO,EAAQ,kBAAkB,GAAK,CAACP,EAAG,kBACtC,OAAOQ,EAGT,GAAI,CAACR,EAAG,OAAQ,CACd,IAAMK,EAAUH,EAAW,cAAeF,CAAE,EACxC,UAAUK,EAAQ,SAAS,IAAM,SAAS,YAAY,IACxDL,EAAG,OAAS,GAChB,CAGA,OAAOS,EAAM,IAAM,CACjB,IAAMC,EAAQ,IAAIC,EAClB,OAAAD,EAAM,UAAU,CAAC,CAAE,KAAAE,CAAK,IAAM,CAC5BZ,EAAG,OAAS,GAGZ,SAAiB,aAAcY,CAAI,CACrC,CAAC,EAGMb,GAAcC,CAAE,EACpB,KACCa,EAAIC,GAASJ,EAAM,KAAKI,CAAK,CAAC,EAC9BC,EAAS,IAAML,EAAM,SAAS,CAAC,EAC/BN,EAAIU,GAAUE,EAAA,CAAE,IAAKhB,GAAOc,EAAQ,CACtC,CACJ,CAAC,CACH,CC5BO,SAASG,GACdC,EAAiB,CAAE,QAAAC,CAAQ,EACN,CACrB,OAAOA,EACJ,KACCC,EAAIC,IAAW,CAAE,OAAQA,IAAWH,CAAG,EAAE,CAC3C,CACJ,CAYO,SAASI,GACdJ,EAAiBK,EACe,CAChC,IAAMC,EAAY,IAAIC,EACtB,OAAAD,EAAU,UAAU,CAAC,CAAE,OAAAE,CAAO,IAAM,CAClCR,EAAG,OAASQ,CACd,CAAC,EAGMT,GAAaC,EAAIK,CAAO,EAC5B,KACCI,EAAIC,GAASJ,EAAU,KAAKI,CAAK,CAAC,EAClCC,EAAS,IAAML,EAAU,SAAS,CAAC,EACnCJ,EAAIQ,GAAUE,EAAA,CAAE,IAAKZ,GAAOU,EAAQ,CACtC,CACJ,CCrEO,SAASG,GACdC,EAAaC,EACA,CACb,OAAIA,IAAU,SAEVC,EAAC,OAAI,MAAM,gCAAgC,GAAIF,EAAI,KAAK,WACtDE,EAAC,OAAI,MAAM,+BAA+B,CAC5C,EAIAA,EAAC,OAAI,MAAM,aAAa,GAAIF,EAAI,KAAK,WACnCE,EAAC,OAAI,MAAM,+BAA+B,CAC5C,CAGN,CCxBO,SAASC,GACdC,EAAqBC,EACR,CAIb,GAHAA,EAASA,EAAS,GAAGA,CAAM,eAAeD,CAAE,GAAK,OAG7CC,EAAQ,CACV,IAAMC,EAASD,EAAS,IAAIA,CAAM,GAAK,OACvC,OACEE,EAAC,SAAM,MAAM,gBAAgB,SAAU,GACpCC,GAAcH,CAAM,EACrBE,EAAC,KAAE,KAAMD,EAAQ,MAAM,uBAAuB,SAAU,IACtDC,EAAC,QAAK,wBAAuBH,EAAI,CACnC,CACF,CAEJ,KACE,QACEG,EAAC,SAAM,MAAM,gBAAgB,SAAU,GACpCC,GAAcH,CAAM,EACrBE,EAAC,QAAK,MAAM,uBAAuB,SAAU,IAC3CA,EAAC,QAAK,wBAAuBH,EAAI,CACnC,CACF,CAGN,CC5BO,SAASK,GAAsBC,EAAyB,CAC7D,OACEC,EAAC,UACC,MAAM,uBACN,MAAOC,GAAY,gBAAgB,EACnC,wBAAuB,IAAIF,CAAE,UAC9B,CAEL,CCQA,SAASG,GACPC,EAAsBC,EACT,CACb,IAAMC,EAASD,EAAO,EAChBE,EAASF,EAAO,EAGhBG,EAAU,OAAO,KAAKJ,EAAS,KAAK,EACvC,OAAOK,GAAO,CAACL,EAAS,MAAMK,CAAG,CAAC,EAClC,OAAyB,CAACC,EAAMD,IAAQ,CACvC,GAAGC,EAAMC,EAAC,WAAKF,CAAI,EAAQ,GAC7B,EAAG,CAAC,CAAC,EACJ,MAAM,EAAG,EAAE,EAGRG,EAASC,GAAc,EACvBC,EAAM,IAAI,IAAIV,EAAS,SAAUQ,EAAO,IAAI,EAC9CG,EAAQ,kBAAkB,GAC5BD,EAAI,aAAa,IAAI,IAAK,OAAO,QAAQV,EAAS,KAAK,EACpD,OAAO,CAAC,CAAC,CAAEY,CAAK,IAAMA,CAAK,EAC3B,OAAO,CAACC,EAAW,CAACC,CAAK,IAAM,GAAGD,CAAS,IAAIC,CAAK,GAAG,KAAK,EAAG,EAAE,CACpE,EAGF,GAAM,CAAE,KAAAC,CAAK,EAAIN,GAAc,EAC/B,OACEF,EAAC,KAAE,KAAM,GAAGG,CAAG,GAAI,MAAM,yBAAyB,SAAU,IAC1DH,EAAC,WACC,MAAM,uCACN,gBAAeP,EAAS,MAAM,QAAQ,CAAC,GAEtCE,EAAS,GAAKK,EAAC,OAAI,MAAM,iCAAiC,EAC1DL,EAAS,GAAKK,EAAC,UAAIP,EAAS,KAAM,EAClCE,GAAU,GAAKK,EAAC,UAAIP,EAAS,KAAM,EACnCG,EAAS,GAAKH,EAAS,KAAK,OAAS,GACpCA,EAAS,KAEVA,EAAS,MAAQA,EAAS,KAAK,IAAIgB,GAAO,CACzC,IAAMC,EAAOF,EACTC,KAAOD,EACL,uBAAuBA,EAAKC,CAAG,CAAC,GAChC,cACF,GACJ,OACET,EAAC,QAAK,MAAO,UAAUU,CAAI,IAAKD,CAAI,CAExC,CAAC,EACAb,EAAS,GAAKC,EAAQ,OAAS,GAC9BG,EAAC,KAAE,MAAM,2BACNW,GAAY,4BAA4B,EAAE,KAAG,GAAGd,CACnD,CAEJ,CACF,CAEJ,CAaO,SAASe,GACdC,EACa,CACb,IAAMC,EAAYD,EAAO,CAAC,EAAE,MACtBE,EAAO,CAAC,GAAGF,CAAM,EAEjBZ,EAASC,GAAc,EAGvBP,EAASoB,EAAK,UAAUC,GAErB,CADG,GAAG,IAAI,IAAIA,EAAI,SAAUf,EAAO,IAAI,CAAC,GACrC,SAAS,GAAG,CACvB,EACK,CAACgB,CAAO,EAAIF,EAAK,OAAOpB,EAAQ,CAAC,EAGnCuB,EAAQH,EAAK,UAAUC,GAAOA,EAAI,MAAQF,CAAS,EACnDI,IAAU,KACZA,EAAQH,EAAK,QAGf,IAAMI,EAAOJ,EAAK,MAAM,EAAGG,CAAK,EAC1BE,EAAOL,EAAK,MAAMG,CAAK,EAGvBG,EAAW,CACf7B,GAAqByB,EAAS,EAAc,EAAE,CAACtB,GAAUuB,IAAU,EAAE,EACrE,GAAGC,EAAK,IAAIG,GAAW9B,GAAqB8B,EAAS,CAAW,CAAC,EACjE,GAAGF,EAAK,OAAS,CACfpB,EAAC,WAAQ,MAAM,0BACbA,EAAC,WAAQ,SAAU,IACjBA,EAAC,WACEoB,EAAK,OAAS,GAAKA,EAAK,SAAW,EAChCT,GAAY,wBAAwB,EACpCA,GAAY,2BAA4BS,EAAK,MAAM,CAEzD,CACF,EACC,GAAGA,EAAK,IAAIE,GAAW9B,GAAqB8B,EAAS,CAAW,CAAC,CACpE,CACF,EAAI,CAAC,CACP,EAGA,OACEtB,EAAC,MAAG,MAAM,0BACPqB,CACH,CAEJ,CCrIO,SAASE,GAAkBC,EAAiC,CACjE,OACEC,EAAC,MAAG,MAAM,oBACP,OAAO,QAAQD,CAAK,EAAE,IAAI,CAAC,CAACE,EAAKC,CAAK,IACrCF,EAAC,MAAG,MAAO,oCAAoCC,CAAG,IAC/C,OAAOC,GAAU,SAAWC,GAAMD,CAAK,EAAIA,CAC9C,CACD,CACH,CAEJ,CCAO,SAASE,GACdC,EACa,CACb,IAAMC,EAAU,kCAAkCD,CAAI,GACtD,OACEE,EAAC,OAAI,MAAOD,EAAS,OAAM,IACzBC,EAAC,UAAO,MAAM,gBAAgB,SAAU,GAAI,cAAY,OAAO,CACjE,CAEJ,CCpBO,SAASC,GAAYC,EAAiC,CAC3D,OACEC,EAAC,OAAI,MAAM,0BACTA,EAAC,OAAI,MAAM,qBACRD,CACH,CACF,CAEJ,CCMA,SAASE,GAAcC,EAA+B,CACpD,IAAMC,EAASC,GAAc,EAGvBC,EAAM,IAAI,IAAI,MAAMH,EAAQ,OAAO,IAAKC,EAAO,IAAI,EACzD,OACEG,EAAC,MAAG,MAAM,oBACRA,EAAC,KAAE,KAAM,GAAGD,CAAG,GAAI,MAAM,oBACtBH,EAAQ,KACX,CACF,CAEJ,CAcO,SAASK,GACdC,EAAqBC,EACR,CACb,OACEH,EAAC,OAAI,MAAM,cACTA,EAAC,UACC,MAAM,sBACN,aAAYI,GAAY,gBAAgB,GAEvCD,EAAO,KACV,EACAH,EAAC,MAAG,MAAM,oBACPE,EAAS,IAAIP,EAAa,CAC7B,CACF,CAEJ,CCfA,IAAIU,GAAW,EAiBR,SAASC,GACdC,EAAiBC,EACI,CACrB,SAAS,KAAK,OAAOD,CAAE,EAGvB,GAAM,CAAE,MAAAE,CAAM,EAAIC,GAAeH,CAAE,EACnCA,EAAG,MAAM,YAAY,qBAAsB,GAAGE,CAAK,IAAI,EACvDF,EAAG,OAAO,EAGV,IAAMI,EAAYC,GAAoBJ,CAAI,EACpCK,EACJ,OAAOF,GAAc,YACjBG,GAA0BH,CAAS,EACnCI,EAAG,CAAE,EAAG,EAAG,EAAG,CAAE,CAAC,EAGjBC,EAAUC,EACdC,GAAkBV,CAAI,EACtBW,GAAkBX,CAAI,CACxB,EACG,KACCY,EAAqB,CACvB,EAGF,OAAOC,EAAc,CAACL,EAASH,CAAO,CAAC,EACpC,KACCS,EAAI,CAAC,CAACC,EAAQC,CAAM,IAAM,CACxB,GAAI,CAAE,EAAAC,EAAG,EAAAC,CAAE,EAAIC,GAAiBnB,CAAI,EAC9BoB,EAAOlB,GAAeF,CAAI,EAU1BqB,EAAQrB,EAAK,QAAQ,OAAO,EAClC,OAAIqB,GAASrB,EAAK,gBAChBiB,GAAKI,EAAM,WAAarB,EAAK,cAAc,WAC3CkB,GAAKG,EAAM,UAAarB,EAAK,cAAc,WAEtC,CACL,OAAAe,EACA,OAAQ,CACN,EAAGE,EAAID,EAAO,EAAII,EAAK,MAAS,EAAInB,EAAQ,EAC5C,EAAGiB,EAAIF,EAAO,EAAII,EAAK,OAAS,CAClC,CACF,CACF,CAAC,CACH,CACJ,CASO,SAASE,GACdvB,EACgC,CAChC,IAAMwB,EAAQxB,EAAG,MACjB,GAAI,CAACwB,EAAM,OACT,OAAOC,EAGT,IAAMC,EAAK,aAAa5B,IAAU,GAC5B6B,EAAUC,GAAcF,EAAI,QAAQ,EACpCG,EAAUC,EAAW,cAAeH,CAAO,EACjD,OAAAE,EAAQ,UAAYL,EAGbO,EAAM,IAAM,CACjB,IAAMC,EAAQ,IAAIC,EAClB,OAAAD,EAAM,UAAU,CAGd,KAAK,CAAE,OAAAE,CAAO,EAAG,CACfP,EAAQ,MAAM,YAAY,iBAAkB,GAAGO,EAAO,CAAC,IAAI,EAC3DP,EAAQ,MAAM,YAAY,iBAAkB,GAAGO,EAAO,CAAC,IAAI,CAC7D,EAGA,UAAW,CACTP,EAAQ,MAAM,eAAe,gBAAgB,EAC7CA,EAAQ,MAAM,eAAe,gBAAgB,CAC/C,CACF,CAAC,EAGDjB,EACEsB,EAAM,KAAKG,EAAO,CAAC,CAAE,OAAAnB,CAAO,IAAMA,CAAM,CAAC,EACzCgB,EAAM,KAAKI,GAAa,GAAG,EAAGD,EAAO,CAAC,CAAE,OAAAnB,CAAO,IAAM,CAACA,CAAM,CAAC,CAC/D,EACG,UAAU,CAGT,KAAK,CAAE,OAAAA,CAAO,EAAG,CACXA,GACFhB,EAAG,sBAAsB,WAAY2B,CAAO,EAC5C3B,EAAG,aAAa,mBAAoB0B,CAAE,EACtC1B,EAAG,gBAAgB,OAAO,IAE1B2B,EAAQ,OAAO,EACf3B,EAAG,gBAAgB,kBAAkB,EACrCA,EAAG,aAAa,QAASwB,CAAK,EAElC,EAGA,UAAW,CACTG,EAAQ,OAAO,EACf3B,EAAG,gBAAgB,kBAAkB,EACrCA,EAAG,aAAa,QAASwB,CAAK,CAChC,CACF,CAAC,EAGHQ,EACG,KACCK,GAAU,GAAIC,EAAuB,CACvC,EACG,UAAU,CAAC,CAAE,OAAAtB,CAAO,IAAM,CACzBW,EAAQ,UAAU,OAAO,qBAAsBX,CAAM,CACvD,CAAC,EAMLgB,EACG,KACCO,GAAa,IAAKD,EAAuB,EACzCH,EAAO,IAAM,CAAC,CAACnC,EAAG,YAAY,EAC9Be,EAAI,IAAMf,EAAG,aAAc,sBAAsB,CAAC,EAClDe,EAAI,CAAC,CAAE,EAAAG,CAAE,IAAMA,CAAC,CAClB,EACC,UAAU,CAGT,KAAKsB,EAAQ,CACPA,EACFb,EAAQ,MAAM,YAAY,iBAAkB,GAAG,CAACa,CAAM,IAAI,EAE1Db,EAAQ,MAAM,eAAe,gBAAgB,CACjD,EAGA,UAAW,CACTA,EAAQ,MAAM,eAAe,gBAAgB,CAC/C,CACF,CAAC,EAGI5B,GAAa4B,EAAS3B,CAAE,EAC5B,KACCyC,EAAIC,GAASV,EAAM,KAAKU,CAAK,CAAC,EAC9BC,EAAS,IAAMX,EAAM,SAAS,CAAC,EAC/BjB,EAAI2B,GAAUE,EAAA,CAAE,IAAK5C,GAAO0C,EAAQ,CACtC,CACJ,CAAC,EACE,KACCG,GAAYC,EAAc,CAC5B,CACJ,CC1KO,SAASC,GACdC,EAAiBC,EACO,CACxB,IAAMC,EAAUC,EAAM,IAAMC,EAAc,CACxCC,GAAmBL,CAAE,EACrBM,GAA0BL,CAAS,CACrC,CAAC,CAAC,EACC,KACCM,EAAI,CAAC,CAAC,CAAE,EAAAC,EAAG,EAAAC,CAAE,EAAGC,CAAM,IAAqB,CACzC,GAAM,CAAE,MAAAC,EAAO,OAAAC,CAAO,EAAIC,GAAeb,CAAE,EAC3C,MAAQ,CACN,EAAGQ,EAAIE,EAAO,EAAIC,EAAS,EAC3B,EAAGF,EAAIC,EAAO,EAAIE,EAAS,CAC7B,CACF,CAAC,CACH,EAGF,OAAOE,GAAkBd,CAAE,EACxB,KACCe,EAAUC,GAAUd,EACjB,KACCK,EAAIU,IAAW,CAAE,OAAAD,EAAQ,OAAAC,CAAO,EAAE,EAClCC,GAAK,CAAC,CAACF,GAAU,GAAQ,CAC3B,CACF,CACF,CACJ,CAWO,SAASG,GACdnB,EAAiBC,EAAwB,CAAE,QAAAmB,CAAQ,EAChB,CACnC,GAAM,CAACC,EAASC,CAAK,EAAI,MAAM,KAAKtB,EAAG,QAAQ,EAG/C,OAAOG,EAAM,IAAM,CACjB,IAAMoB,EAAQ,IAAIC,EACZC,EAAQF,EAAM,KAAKG,GAAe,EAAGC,GAAQ,EAAI,CAAC,EACxD,OAAAJ,EAAM,UAAU,CAGd,KAAK,CAAE,OAAAN,CAAO,EAAG,CACfjB,EAAG,MAAM,YAAY,iBAAkB,GAAGiB,EAAO,CAAC,IAAI,EACtDjB,EAAG,MAAM,YAAY,iBAAkB,GAAGiB,EAAO,CAAC,IAAI,CACxD,EAGA,UAAW,CACTjB,EAAG,MAAM,eAAe,gBAAgB,EACxCA,EAAG,MAAM,eAAe,gBAAgB,CAC1C,CACF,CAAC,EAGD4B,GAAuB5B,CAAE,EACtB,KACC6B,EAAUJ,CAAK,CACjB,EACG,UAAUK,GAAW,CACpB9B,EAAG,gBAAgB,kBAAmB8B,CAAO,CAC/C,CAAC,EAGLC,EACER,EAAM,KAAKS,EAAO,CAAC,CAAE,OAAAhB,CAAO,IAAMA,CAAM,CAAC,EACzCO,EAAM,KAAKU,GAAa,GAAG,EAAGD,EAAO,CAAC,CAAE,OAAAhB,CAAO,IAAM,CAACA,CAAM,CAAC,CAC/D,EACG,UAAU,CAGT,KAAK,CAAE,OAAAA,CAAO,EAAG,CACXA,EACFhB,EAAG,QAAQqB,CAAO,EAElBA,EAAQ,OAAO,CACnB,EAGA,UAAW,CACTrB,EAAG,QAAQqB,CAAO,CACpB,CACF,CAAC,EAGHE,EACG,KACCW,GAAU,GAAIC,EAAuB,CACvC,EACG,UAAU,CAAC,CAAE,OAAAnB,CAAO,IAAM,CACzBK,EAAQ,UAAU,OAAO,qBAAsBL,CAAM,CACvD,CAAC,EAGLO,EACG,KACCa,GAAa,IAAKD,EAAuB,EACzCH,EAAO,IAAM,CAAC,CAAChC,EAAG,YAAY,EAC9BO,EAAI,IAAMP,EAAG,aAAc,sBAAsB,CAAC,EAClDO,EAAI,CAAC,CAAE,EAAAC,CAAE,IAAMA,CAAC,CAClB,EACG,UAAU,CAGT,KAAK6B,EAAQ,CACPA,EACFrC,EAAG,MAAM,YAAY,iBAAkB,GAAG,CAACqC,CAAM,IAAI,EAErDrC,EAAG,MAAM,eAAe,gBAAgB,CAC5C,EAGA,UAAW,CACTA,EAAG,MAAM,eAAe,gBAAgB,CAC1C,CACF,CAAC,EAGLsC,EAAsBhB,EAAO,OAAO,EACjC,KACCO,EAAUJ,CAAK,EACfO,EAAOO,GAAM,EAAEA,EAAG,SAAWA,EAAG,QAAQ,CAC1C,EACG,UAAUA,GAAM,CACfA,EAAG,gBAAgB,EACnBA,EAAG,eAAe,CACpB,CAAC,EAGLD,EAAsBhB,EAAO,WAAW,EACrC,KACCO,EAAUJ,CAAK,EACfe,GAAejB,CAAK,CACtB,EACG,UAAU,CAAC,CAACgB,EAAI,CAAE,OAAAvB,CAAO,CAAC,IAAM,CA3OzC,IAAAyB,EA8OU,GAAIF,EAAG,SAAW,GAAKA,EAAG,SAAWA,EAAG,QACtCA,EAAG,eAAe,UAGTvB,EAAQ,CACjBuB,EAAG,eAAe,EAGlB,IAAMG,EAAS1C,EAAG,cAAe,QAAQ,gBAAgB,EACrD0C,aAAkB,YACpBA,EAAO,MAAM,GAEbD,EAAAE,GAAiB,IAAjB,MAAAF,EAAoB,MACxB,CACF,CAAC,EAGLrB,EACG,KACCS,EAAUJ,CAAK,EACfO,EAAOY,GAAUA,IAAWvB,CAAO,EACnCwB,GAAM,GAAG,CACX,EACG,UAAU,IAAM7C,EAAG,MAAM,CAAC,EAGxBD,GAAgBC,EAAIC,CAAS,EACjC,KACC6C,EAAIC,GAASxB,EAAM,KAAKwB,CAAK,CAAC,EAC9BC,EAAS,IAAMzB,EAAM,SAAS,CAAC,EAC/BhB,EAAIwC,GAAUE,EAAA,CAAE,IAAKjD,GAAO+C,EAAQ,CACtC,CACJ,CAAC,CACH,CCxMA,SAASG,GAAUC,EAAuC,CACxD,OAAOA,EAAU,UAAY,OACzBC,EAAY,eAAgBD,CAAS,EACrC,CAACA,CAAS,CAChB,CASA,SAASE,GAAYF,EAAgC,CACnD,IAAMG,EAAkB,CAAC,EACzB,QAAWC,KAAML,GAAUC,CAAS,EAAG,CACrC,IAAMK,EAAgB,CAAC,EAGjBC,EAAK,SAAS,mBAAmBF,EAAI,WAAW,SAAS,EAC/D,QAASG,EAAOD,EAAG,SAAS,EAAGC,EAAMA,EAAOD,EAAG,SAAS,EACtDD,EAAM,KAAKE,CAAY,EAGzB,QAASC,KAAQH,EAAO,CACtB,IAAII,EAGJ,KAAQA,EAAQ,gBAAgB,KAAKD,EAAK,WAAY,GAAI,CACxD,GAAM,CAAC,CAAEE,EAAIC,CAAK,EAAIF,EACtB,GAAI,OAAOE,GAAU,YAAa,CAChC,IAAMC,EAASJ,EAAK,UAAUC,EAAM,KAAK,EACzCD,EAAOI,EAAO,UAAUF,EAAG,MAAM,EACjCP,EAAQ,KAAKS,CAAM,CAGrB,KAAO,CACLJ,EAAK,YAAcE,EACnBP,EAAQ,KAAKK,CAAI,EACjB,KACF,CACF,CACF,CACF,CACA,OAAOL,CACT,CAQA,SAASU,GAAKC,EAAqBC,EAA2B,CAC5DA,EAAO,OAAO,GAAG,MAAM,KAAKD,EAAO,UAAU,CAAC,CAChD,CAoBO,SAASE,GACdZ,EAAiBJ,EAAwB,CAAE,QAAAiB,EAAS,OAAAC,CAAO,EACxB,CAGnC,IAAMC,EAASnB,EAAU,QAAQ,MAAM,EACjCoB,EAASD,GAAA,YAAAA,EAAQ,GAGjBE,EAAc,IAAI,IACxB,QAAWT,KAAUV,GAAYF,CAAS,EAAG,CAC3C,GAAM,CAAC,CAAEU,CAAE,EAAIE,EAAO,YAAa,MAAM,WAAW,EAChDU,GAAmB,yBAAyBZ,CAAE,IAAKN,CAAE,IACvDiB,EAAY,IAAIX,EAAIa,GAAiBb,EAAIU,CAAM,CAAC,EAChDR,EAAO,YAAYS,EAAY,IAAIX,CAAE,CAAE,EAE3C,CAGA,OAAIW,EAAY,OAAS,EAChBG,EAGFC,EAAM,IAAM,CACjB,IAAMC,EAAQ,IAAIC,EACZC,EAAQF,EAAM,KAAKG,GAAe,EAAGC,GAAQ,EAAI,CAAC,EAGlDC,EAAsC,CAAC,EAC7C,OAAW,CAACrB,EAAIsB,CAAU,IAAKX,EAC7BU,EAAM,KAAK,CACTE,EAAW,cAAeD,CAAU,EACpCC,EAAW,yBAAyBvB,CAAE,IAAKN,CAAE,CAC/C,CAAC,EAGH,OAAAc,EAAO,KAAKgB,EAAUN,CAAK,CAAC,EACzB,UAAUO,GAAU,CACnB/B,EAAG,OAAS,CAAC+B,EAGb/B,EAAG,UAAU,OAAO,qBAAsB+B,CAAM,EAGhD,OAAW,CAACC,EAAOC,CAAK,IAAKN,EACtBI,EAGHtB,GAAKuB,EAAOC,CAAK,EAFjBxB,GAAKwB,EAAOD,CAAK,CAGvB,CAAC,EAGIE,EAAM,GAAG,CAAC,GAAGjB,CAAW,EAC5B,IAAI,CAAC,CAAC,CAAEW,CAAU,IACjBO,GAAgBP,EAAYhC,EAAW,CAAE,QAAAiB,CAAQ,CAAC,CACnD,CACH,EACG,KACCuB,EAAS,IAAMd,EAAM,SAAS,CAAC,EAC/Be,GAAM,CACR,CACJ,CAAC,CACH,CC7JA,SAASC,GAASC,EAA0C,CAC1D,GAAIA,EAAG,mBAAoB,CACzB,IAAMC,EAAUD,EAAG,mBACnB,GAAIC,EAAQ,UAAY,KACtB,OAAOA,EAGJ,GAAIA,EAAQ,UAAY,KAAO,CAACA,EAAQ,SAAS,OACpD,OAAOF,GAASE,CAAO,CAC3B,CAIF,CAcO,SAASC,GACdF,EAAiBG,EACkB,CACnC,OAAOC,EAAM,IAAM,CACjB,IAAMC,EAAON,GAASC,CAAE,EACxB,OAAO,OAAOK,GAAS,YACnBC,GAAoBD,EAAML,EAAIG,CAAO,EACrCI,CACN,CAAC,CACH,CCjEA,IAAAC,GAAwB,SA2ExB,IAAIC,GAAW,EAaf,SAASC,GAAkBC,EAA0C,CACnE,GAAIA,EAAG,mBAAoB,CACzB,IAAMC,EAAUD,EAAG,mBACnB,GAAIC,EAAQ,UAAY,KACtB,OAAOA,EAGJ,GAAIA,EAAQ,UAAY,KAAO,CAACA,EAAQ,SAAS,OACpD,OAAOF,GAAkBE,CAAO,CACpC,CAIF,CAgBO,SAASC,GACdF,EACsB,CACtB,OAAOG,GAAiBH,CAAE,EACvB,KACCI,EAAI,CAAC,CAAE,MAAAC,CAAM,KAEJ,CACL,WAFcC,GAAsBN,CAAE,EAElB,MAAQK,CAC9B,EACD,EACDE,EAAwB,YAAY,CACtC,CACJ,CAoBO,SAASC,GACdR,EAAiBS,EACiB,CAClC,GAAM,CAAE,QAASC,CAAM,EAAI,WAAW,SAAS,EAGzCC,EAAWC,EAAM,IAAM,CAC3B,IAAMC,EAAQ,IAAIC,EACZC,EAAQF,EAAM,KAAKG,GAAS,CAAC,CAAC,EACpCH,EAAM,UAAU,CAAC,CAAE,WAAAI,CAAW,IAAM,CAC9BA,GAAcP,EAChBV,EAAG,aAAa,WAAY,GAAG,EAE/BA,EAAG,gBAAgB,UAAU,CACjC,CAAC,EAGD,IAAMkB,EAAoD,CAAC,EAC3D,GAAI,GAAAC,QAAY,YAAY,IACtBnB,EAAG,QAAQ,OAAO,GACpBoB,EAAQ,mBAAmB,GAAK,CAACpB,EAAG,QAAQ,UAAU,GACrD,CACD,IAAMqB,EAASrB,EAAG,QAAQ,KAAK,EAC/BqB,EAAO,GAAK,UAAUvB,IAAU,GAGhC,IAAMwB,EAASC,GAAsBF,EAAO,EAAE,EAC9CA,EAAO,aAAaC,EAAQtB,CAAE,EAC1BoB,EAAQ,kBAAkB,GAC5BF,EAAS,KAAKM,GAAaF,CAAM,CAAC,CACtC,CAIF,IAAMG,EAAYzB,EAAG,QAAQ,YAAY,EACzC,GAAIyB,aAAqB,YAAa,CACpC,IAAMC,EAAO3B,GAAkB0B,CAAS,EAGxC,GAAI,OAAOC,GAAS,cAClBD,EAAU,UAAU,SAAS,UAAU,GACvCL,EAAQ,uBAAuB,GAC9B,CACD,IAAMO,EAAeC,GAAoBF,EAAM1B,EAAIS,CAAO,EAC1DS,EAAS,KACPf,GAAiBsB,CAAS,EACvB,KACCI,EAAUd,CAAK,EACfX,EAAI,CAAC,CAAE,MAAAC,EAAO,OAAAyB,CAAO,IAAMzB,GAASyB,CAAM,EAC1CC,EAAqB,EACrBC,EAAUC,GAAUA,EAASN,EAAeO,CAAK,CACnD,CACJ,CACF,CACF,CAGA,OAAOhC,GAAeF,CAAE,EACrB,KACCmC,EAAIC,GAASvB,EAAM,KAAKuB,CAAK,CAAC,EAC9BC,EAAS,IAAMxB,EAAM,SAAS,CAAC,EAC/BT,EAAIgC,GAAUE,EAAA,CAAE,IAAKtC,GAAOoC,EAAQ,EACpCG,GAAU,GAAGrB,CAAQ,CACvB,CACJ,CAAC,EAGD,OAAIE,EAAQ,cAAc,EACjBoB,GAAuBxC,CAAE,EAC7B,KACCyC,EAAOC,GAAWA,CAAO,EACzBC,GAAK,CAAC,EACNX,EAAU,IAAMrB,CAAQ,CAC1B,EAGGA,CACT,CC1KO,SAASiC,GACdC,EAAwB,CAAE,QAAAC,EAAS,OAAAC,CAAO,EACrB,CACrB,IAAIC,EAAO,GACX,OAAOC,EAGLH,EACG,KACCI,EAAIC,GAAUA,EAAO,QAAQ,qBAAqB,CAAE,EACpDC,EAAOC,GAAWR,IAAOQ,CAAO,EAChCH,EAAI,KAAO,CACT,OAAQ,OAAQ,OAAQ,EAC1B,EAAa,CACf,EAGFH,EACG,KACCK,EAAOE,GAAUA,GAAU,CAACN,CAAI,EAChCO,EAAI,IAAMP,EAAOH,EAAG,IAAI,EACxBK,EAAII,IAAW,CACb,OAAQA,EAAS,OAAS,OAC5B,EAAa,CACf,CACJ,CACF,CAaO,SAASE,GACdX,EAAwBY,EACQ,CAChC,OAAOC,EAAM,IAAM,CACjB,IAAMC,EAAQ,IAAIC,EAClB,OAAAD,EAAM,UAAU,CAAC,CAAE,OAAAE,EAAQ,OAAAC,CAAO,IAAM,CACtCjB,EAAG,gBAAgB,OAAQgB,IAAW,MAAM,EACxCC,GACFjB,EAAG,eAAe,CACtB,CAAC,EAGMD,GAAaC,EAAIY,CAAO,EAC5B,KACCF,EAAIQ,GAASJ,EAAM,KAAKI,CAAK,CAAC,EAC9BC,EAAS,IAAML,EAAM,SAAS,CAAC,EAC/BT,EAAIa,GAAUE,EAAA,CAAE,IAAKpB,GAAOkB,EAAQ,CACtC,CACJ,CAAC,CACH,CCzIA,IAAAG,GAAA,ktLCqDA,IAAIC,GAKAC,GAAW,EAWf,SAASC,IAAiC,CACxC,OAAO,OAAO,SAAY,aAAe,mBAAmB,QACxDC,GAAY,sDAAsD,EAClEC,EAAG,MAAS,CAClB,CAaO,SAASC,GACdC,EACgC,CAChC,OAAAA,EAAG,UAAU,OAAO,SAAS,EAC7BN,QAAaE,GAAa,EACvB,KACCK,EAAI,IAAM,QAAQ,WAAW,CAC3B,YAAa,GACb,SAAAC,GACA,SAAU,CACR,cAAe,OACf,gBAAiB,OACjB,aAAc,MAChB,CACF,CAAC,CAAC,EACFC,EAAI,IAAG,EAAY,EACnBC,EAAY,CAAC,CACf,GAGFV,GAAS,UAAU,IAAYW,GAAA,sBAC7BL,EAAG,UAAU,IAAI,SAAS,EAC1B,IAAMM,EAAK,aAAaX,IAAU,GAG5BY,EAAOC,EAAE,MAAO,CAAE,MAAO,SAAU,CAAC,EACpCC,EAAOT,EAAG,YAGV,CAAE,IAAAU,EAAK,GAAAC,CAAG,EAAI,MAAM,QAAQ,OAAOL,EAAIG,CAAI,EAG3CG,EAASL,EAAK,aAAa,CAAE,KAAM,QAAS,CAAC,EACnDK,EAAO,UAAYF,EAGnBV,EAAG,YAAYO,CAAI,EACnBI,GAAA,MAAAA,EAAKC,EACP,EAAC,EAGMlB,GACJ,KACCS,EAAI,KAAO,CAAE,IAAKH,CAAG,EAAE,CACzB,CACJ,CCtFA,IAAMa,GAAWC,EAAE,OAAO,EAgBnB,SAASC,GACdC,EACkC,CAClC,OAAAA,EAAG,YAAYH,EAAQ,EACvBA,GAAS,YAAYI,GAAYD,CAAE,CAAC,EAG7BE,EAAG,CAAE,IAAKF,CAAG,CAAC,CACvB,CC2BO,SAASG,GACdC,EACyB,CACzB,IAAMC,EAAUD,EAAO,KAAKE,GAASA,EAAM,OAAO,GAAKF,EAAO,CAAC,EAC/D,OAAOG,EAAM,GAAGH,EAAO,IAAIE,GAASE,EAAUF,EAAO,QAAQ,EAC1D,KACCG,EAAI,IAAMC,EAA6B,cAAcJ,EAAM,EAAE,IAAI,CAAC,CACpE,CACF,CAAC,EACE,KACCK,EAAUD,EAA6B,cAAcL,EAAQ,EAAE,IAAI,CAAC,EACpEI,EAAIG,IAAW,CAAE,OAAAA,CAAO,EAAE,CAC5B,CACJ,CAUO,SAASC,GACdC,EAAiB,CAAE,UAAAC,EAAW,QAAAC,CAAQ,EACF,CACpC,IAAMC,EAAYP,EAAW,iBAAkBI,CAAE,EAC3CV,EAASc,EAA8B,iBAAkBJ,CAAE,EAG3DK,EAAOC,GAAoB,MAAM,EACvCN,EAAG,OAAOK,CAAI,EAGd,IAAME,EAAOD,GAAoB,MAAM,EACvC,OAAAN,EAAG,OAAOO,CAAI,EAGPC,EAAM,IAAM,CACjB,IAAMC,EAAQ,IAAIC,EACZC,EAAQF,EAAM,KAAKG,GAAe,EAAGC,GAAQ,EAAI,CAAC,EACxDC,EAAc,CAACL,EAAOM,GAAiBf,CAAE,CAAC,CAAC,EACxC,KACCgB,EAAUL,CAAK,EACfM,GAAU,EAAGC,EAAuB,CACtC,EACG,UAAU,CAGT,KAAK,CAAC,CAAE,OAAApB,CAAO,EAAGqB,CAAI,EAAG,CACvB,IAAMC,EAASC,GAAiBvB,CAAM,EAChC,CAAE,MAAAwB,CAAM,EAAIC,GAAezB,CAAM,EAGvCE,EAAG,MAAM,YAAY,mBAAoB,GAAGoB,EAAO,CAAC,IAAI,EACxDpB,EAAG,MAAM,YAAY,uBAAwB,GAAGsB,CAAK,IAAI,EAGzD,IAAME,EAAUC,GAAwBtB,CAAS,GAE/CiB,EAAO,EAAYI,EAAQ,GAC3BJ,EAAO,EAAIE,EAAQE,EAAQ,EAAIL,EAAK,QAEpChB,EAAU,SAAS,CACjB,KAAM,KAAK,IAAI,EAAGiB,EAAO,EAAI,EAAE,EAC/B,SAAU,QACZ,CAAC,CACL,EAGA,UAAW,CACTpB,EAAG,MAAM,eAAe,kBAAkB,EAC1CA,EAAG,MAAM,eAAe,sBAAsB,CAChD,CACF,CAAC,EAGLc,EAAc,CACZY,GAA0BvB,CAAS,EACnCY,GAAiBZ,CAAS,CAC5B,CAAC,EACE,KACCa,EAAUL,CAAK,CACjB,EACG,UAAU,CAAC,CAACS,EAAQD,CAAI,IAAM,CAC7B,IAAMK,EAAUG,GAAsBxB,CAAS,EAC/CE,EAAK,OAASe,EAAO,EAAI,GACzBb,EAAK,OAASa,EAAO,EAAII,EAAQ,MAAQL,EAAK,MAAQ,EACxD,CAAC,EAGL1B,EACEC,EAAUW,EAAM,OAAO,EAAE,KAAKV,EAAI,IAAM,EAAE,CAAC,EAC3CD,EAAUa,EAAM,OAAO,EAAE,KAAKZ,EAAI,IAAM,CAAE,CAAC,CAC7C,EACG,KACCqB,EAAUL,CAAK,CACjB,EACG,UAAUiB,GAAa,CACtB,GAAM,CAAE,MAAAN,CAAM,EAAIC,GAAepB,CAAS,EAC1CA,EAAU,SAAS,CACjB,KAAMmB,EAAQM,EACd,SAAU,QACZ,CAAC,CACH,CAAC,EAGL1B,EACG,KACCc,EAAUL,CAAK,EACfkB,EAAOrC,GAASF,EAAO,SAASE,CAAyB,CAAC,CAC5D,EACG,UAAUA,GAASA,EAAM,MAAM,CAAC,EAGrCW,EAAU,UAAU,IAAI,uBAAuB,EAC/C,QAAWX,KAASF,EAAQ,CAC1B,IAAMwC,EAAQlC,EAA6B,cAAcJ,EAAM,EAAE,IAAI,EACrEsC,EAAM,gBAAgBC,EAAE,IAAK,CAC3B,KAAM,IAAID,EAAM,OAAO,GACvB,SAAU,EACZ,EAAG,GAAG,MAAM,KAAKA,EAAM,UAAU,CAAC,CAAC,EAGnCpC,EAAsBoC,EAAM,kBAAoB,OAAO,EACpD,KACCd,EAAUL,CAAK,EACfkB,EAAOG,GAAM,EAAEA,EAAG,SAAWA,EAAG,QAAQ,EACxCC,EAAID,GAAM,CACRA,EAAG,eAAe,EAClBA,EAAG,gBAAgB,CACrB,CAAC,CACH,EAEG,UAAU,IAAM,CACf,QAAQ,aAAa,CAAC,EAAG,GAAI,IAAIF,EAAM,OAAO,EAAE,EAChDA,EAAM,MAAM,CACd,CAAC,CACP,CAGA,OAAII,EAAQ,mBAAmB,GAC7BzB,EAAM,KACJ0B,GAAK,CAAC,EACNC,GAAenC,CAAS,CAC1B,EACG,UAAU,CAAC,CAAC,CAAE,OAAAH,CAAO,EAAG,CAAE,OAAAsB,CAAO,CAAC,IAAM,CACvC,IAAMiB,EAAMvC,EAAO,UAAU,KAAK,EAClC,GAAIA,EAAO,aAAa,mBAAmB,EACzCA,EAAO,gBAAgB,mBAAmB,MAGrC,CACL,IAAMwC,EAAItC,EAAG,UAAYoB,EAAO,EAGhC,QAAWmB,KAAOnC,EAAY,aAAa,EACzC,QAAWZ,KAASY,EAClB,iBAAkBmC,CACpB,EAAG,CACD,IAAMT,EAAQlC,EAAW,cAAcJ,EAAM,EAAE,IAAI,EACnD,GACEsC,IAAUhC,GACVgC,EAAM,UAAU,KAAK,IAAMO,EAC3B,CACAP,EAAM,aAAa,oBAAqB,EAAE,EAC1CtC,EAAM,MAAM,EACZ,KACF,CACF,CAGF,OAAO,SAAS,CACd,IAAKQ,EAAG,UAAYsC,CACtB,CAAC,EAGD,IAAME,EAAO,SAAmB,QAAQ,GAAK,CAAC,EAC9C,SAAS,SAAU,CAAC,GAAG,IAAI,IAAI,CAACH,EAAK,GAAGG,CAAI,CAAC,CAAC,CAAC,CACjD,CACF,CAAC,EAGL/B,EAAM,KAAKO,EAAUL,CAAK,CAAC,EACxB,UAAU,IAAM,CACf,QAAW8B,KAASrC,EAA8B,eAAgBJ,CAAE,EAClEyC,EAAM,MAAM,CAChB,CAAC,EAGIpD,GAAiBC,CAAM,EAC3B,KACC2C,EAAIS,GAASjC,EAAM,KAAKiC,CAAK,CAAC,EAC9BC,EAAS,IAAMlC,EAAM,SAAS,CAAC,EAC/Bd,EAAI+C,GAAUE,EAAA,CAAE,IAAK5C,GAAO0C,EAAQ,CACtC,CACJ,CAAC,EACE,KACCG,GAAYC,EAAc,CAC5B,CACJ,CCnMO,SAASC,GACdC,EAAiB,CAAE,UAAAC,EAAW,QAAAC,EAAS,OAAAC,CAAO,EACd,CAChC,OAAOC,EAGL,GAAGC,EAAY,4BAA6BL,CAAE,EAC3C,IAAIM,GAASC,GAAqBD,EAAO,CAAE,QAAAJ,EAAS,OAAAC,CAAO,CAAC,CAAC,EAGhE,GAAGE,EAAY,2BAA4BL,CAAE,EAC1C,IAAIM,GAASE,GAAeF,EAAO,CAAE,QAAAJ,EAAS,OAAAC,CAAO,CAAC,CAAC,EAG1D,GAAGE,EAAY,cAAeL,CAAE,EAC7B,IAAIM,GAASG,GAAaH,CAAK,CAAC,EAGnC,GAAGD,EAAY,qBAAsBL,CAAE,EACpC,IAAIM,GAASI,GAAeJ,CAAK,CAAC,EAGrC,GAAGD,EAAY,UAAWL,CAAE,EACzB,IAAIM,GAASK,GAAaL,EAAO,CAAE,QAAAJ,EAAS,OAAAC,CAAO,CAAC,CAAC,EAGxD,GAAGE,EAAY,cAAeL,CAAE,EAC7B,IAAIM,GAASM,GAAiBN,EAAO,CAAE,UAAAL,EAAW,QAAAC,CAAQ,CAAC,CAAC,EAG/D,GAAGG,EAAY,UAAWL,CAAE,EACzB,OAAO,IAAMa,EAAQ,kBAAkB,CAAC,EACxC,IAAIP,GAASQ,GAAaR,CAAK,CAAC,CACrC,CACF,CCtDO,SAASS,GACdC,EAAkB,CAAE,OAAAC,CAAO,EACP,CACpB,OAAOA,EACJ,KACCC,EAAUC,GAAWC,EACnBC,EAAG,EAAI,EACPA,EAAG,EAAK,EAAE,KAAKC,GAAM,GAAI,CAAC,CAC5B,EACG,KACCC,EAAIC,IAAW,CAAE,QAAAL,EAAS,OAAAK,CAAO,EAAE,CACrC,CACF,CACF,CACJ,CAaO,SAASC,GACdC,EAAiBC,EACc,CAC/B,IAAMC,EAAQC,EAAW,cAAeH,CAAE,EAC1C,OAAOI,EAAM,IAAM,CACjB,IAAMC,EAAQ,IAAIC,EAClB,OAAAD,EAAM,UAAU,CAAC,CAAE,QAAAZ,EAAS,OAAAK,CAAO,IAAM,CACvCE,EAAG,UAAU,OAAO,oBAAqBF,CAAM,EAC/CI,EAAM,YAAcT,CACtB,CAAC,EAGMJ,GAAYW,EAAIC,CAAO,EAC3B,KACCM,EAAIC,GAASH,EAAM,KAAKG,CAAK,CAAC,EAC9BC,EAAS,IAAMJ,EAAM,SAAS,CAAC,EAC/BR,EAAIW,GAAUE,EAAA,CAAE,IAAKV,GAAOQ,EAAQ,CACtC,CACJ,CAAC,CACH,CCrBA,SAASG,GAAS,CAAE,UAAAC,CAAU,EAAsC,CAClE,GAAI,CAACC,EAAQ,iBAAiB,EAC5B,OAAOC,EAAG,EAAK,EAGjB,IAAMC,EAAaH,EAChB,KACCI,EAAI,CAAC,CAAE,OAAQ,CAAE,EAAAC,CAAE,CAAE,IAAMA,CAAC,EAC5BC,GAAY,EAAG,CAAC,EAChBF,EAAI,CAAC,CAACG,EAAGC,CAAC,IAAM,CAACD,EAAIC,EAAGA,CAAC,CAAU,EACnCC,EAAwB,CAAC,CAC3B,EAGIC,EAAUC,EAAc,CAACX,EAAWG,CAAU,CAAC,EAClD,KACCS,EAAO,CAAC,CAAC,CAAE,OAAAC,CAAO,EAAG,CAAC,CAAER,CAAC,CAAC,IAAM,KAAK,IAAIA,EAAIQ,EAAO,CAAC,EAAI,GAAG,EAC5DT,EAAI,CAAC,CAAC,CAAE,CAACU,CAAS,CAAC,IAAMA,CAAS,EAClCC,EAAqB,CACvB,EAGIC,EAAUC,GAAY,QAAQ,EACpC,OAAON,EAAc,CAACX,EAAWgB,CAAO,CAAC,EACtC,KACCZ,EAAI,CAAC,CAAC,CAAE,OAAAS,CAAO,EAAGK,CAAM,IAAML,EAAO,EAAI,KAAO,CAACK,CAAM,EACvDH,EAAqB,EACrBI,EAAUC,GAAUA,EAASV,EAAUR,EAAG,EAAK,CAAC,EAChDmB,EAAU,EAAK,CACjB,CACJ,CAcO,SAASC,GACdC,EAAiBC,EACG,CACpB,OAAOC,EAAM,IAAMd,EAAc,CAC/Be,GAAiBH,CAAE,EACnBxB,GAASyB,CAAO,CAClB,CAAC,CAAC,EACC,KACCpB,EAAI,CAAC,CAAC,CAAE,OAAAuB,CAAO,EAAGC,CAAM,KAAO,CAC7B,OAAAD,EACA,OAAAC,CACF,EAAE,EACFb,EAAqB,CAACR,EAAGC,IACvBD,EAAE,SAAWC,EAAE,QACfD,EAAE,SAAWC,EAAE,MAChB,EACDqB,EAAY,CAAC,CACf,CACJ,CAaO,SAASC,GACdP,EAAiB,CAAE,QAAAQ,EAAS,MAAAC,CAAM,EACO,CACzC,OAAOP,EAAM,IAAM,CACjB,IAAMQ,EAAQ,IAAIC,EACZC,EAAQF,EAAM,KAAKG,GAAe,EAAGC,GAAQ,EAAI,CAAC,EACxDJ,EACG,KACCxB,EAAwB,QAAQ,EAChC6B,GAAkBP,CAAO,CAC3B,EACG,UAAU,CAAC,CAAC,CAAE,OAAAX,CAAO,EAAG,CAAE,OAAAQ,CAAO,CAAC,IAAM,CACvCL,EAAG,UAAU,OAAO,oBAAqBH,GAAU,CAACQ,CAAM,EAC1DL,EAAG,OAASK,CACd,CAAC,EAGL,IAAMW,EAAWC,GAAKC,EAAY,UAAWlB,CAAE,CAAC,EAC7C,KACCX,EAAO,IAAMX,EAAQ,kBAAkB,CAAC,EACxCyC,GAASC,GAASC,GAAaD,CAAK,CAAC,CACvC,EAGF,OAAAX,EAAM,UAAUC,CAAK,EAGdF,EACJ,KACCc,EAAUV,CAAK,EACf/B,EAAI0C,GAAUC,EAAA,CAAE,IAAKxB,GAAOuB,EAAQ,EACpCE,GAAUT,EAAS,KAAKM,EAAUV,CAAK,CAAC,CAAC,CAC3C,CACJ,CAAC,CACH,CCjIO,SAASc,GACdC,EAAiB,CAAE,UAAAC,EAAW,QAAAC,CAAQ,EACb,CACzB,OAAOC,GAAgBH,EAAI,CAAE,UAAAC,EAAW,QAAAC,CAAQ,CAAC,EAC9C,KACCE,EAAI,CAAC,CAAE,OAAQ,CAAE,EAAAC,CAAE,CAAE,IAAM,CACzB,GAAM,CAAE,OAAAC,CAAO,EAAIC,GAAeP,CAAE,EACpC,MAAO,CACL,OAAQK,GAAKC,CACf,CACF,CAAC,EACDE,EAAwB,QAAQ,CAClC,CACJ,CAaO,SAASC,GACdT,EAAiBU,EACmB,CACpC,OAAOC,EAAM,IAAM,CACjB,IAAMC,EAAQ,IAAIC,EAClBD,EAAM,UAAU,CAGd,KAAK,CAAE,OAAAE,CAAO,EAAG,CACfd,EAAG,UAAU,OAAO,2BAA4Bc,CAAM,CACxD,EAGA,UAAW,CACTd,EAAG,UAAU,OAAO,0BAA0B,CAChD,CACF,CAAC,EAGD,IAAMe,EAAUC,GAAmB,gBAAgB,EACnD,OAAI,OAAOD,GAAY,YACdE,EAGFlB,GAAiBgB,EAASL,CAAO,EACrC,KACCQ,EAAIC,GAASP,EAAM,KAAKO,CAAK,CAAC,EAC9BC,EAAS,IAAMR,EAAM,SAAS,CAAC,EAC/BR,EAAIe,GAAUE,EAAA,CAAE,IAAKrB,GAAOmB,EAAQ,CACtC,CACJ,CAAC,CACH,CChEO,SAASG,GACdC,EAAiB,CAAE,UAAAC,EAAW,QAAAC,CAAQ,EACpB,CAGlB,IAAMC,EAAUD,EACb,KACCE,EAAI,CAAC,CAAE,OAAAC,CAAO,IAAMA,CAAM,EAC1BC,EAAqB,CACvB,EAGIC,EAAUJ,EACb,KACCK,EAAU,IAAMC,GAAiBT,CAAE,EAChC,KACCI,EAAI,CAAC,CAAE,OAAAC,CAAO,KAAO,CACnB,IAAQL,EAAG,UACX,OAAQA,EAAG,UAAYK,CACzB,EAAE,EACFK,EAAwB,QAAQ,CAClC,CACF,CACF,EAGF,OAAOC,EAAc,CAACR,EAASI,EAASN,CAAS,CAAC,EAC/C,KACCG,EAAI,CAAC,CAACQ,EAAQ,CAAE,IAAAC,EAAK,OAAAC,CAAO,EAAG,CAAE,OAAQ,CAAE,EAAAC,CAAE,EAAG,KAAM,CAAE,OAAAV,CAAO,CAAE,CAAC,KAChEA,EAAS,KAAK,IAAI,EAAGA,EACjB,KAAK,IAAI,EAAGQ,EAASE,EAAIH,CAAM,EAC/B,KAAK,IAAI,EAAGP,EAASU,EAAID,CAAM,CACnC,EACO,CACL,OAAQD,EAAMD,EACd,OAAAP,EACA,OAAQQ,EAAMD,GAAUG,CAC1B,EACD,EACDT,EAAqB,CAACU,EAAGC,IACvBD,EAAE,SAAWC,EAAE,QACfD,EAAE,SAAWC,EAAE,QACfD,EAAE,SAAWC,EAAE,MAChB,CACH,CACJ,CC1CO,SAASC,GACdC,EACqB,CACrB,IAAMC,EAAU,SAAkB,WAAW,GAAK,CAChD,MAAOD,EAAO,UAAUE,GAAS,WAC/BA,EAAM,aAAa,qBAAqB,CAC1C,EAAE,OAAO,CACX,EAGMC,EAAQ,KAAK,IAAI,EAAG,KAAK,IAAIF,EAAQ,MAAOD,EAAO,OAAS,CAAC,CAAC,EACpE,OAAOI,EAAG,GAAGJ,CAAM,EAChB,KACCK,GAASH,GAASI,EAAUJ,EAAO,QAAQ,EAAE,KAAKK,EAAI,IAAML,CAAK,CAAC,CAAC,EACnEM,EAAUR,EAAOG,CAAK,CAAC,EACvBI,EAAIL,IAAU,CACZ,MAAOF,EAAO,QAAQE,CAAK,EAC3B,MAAO,CACL,MAASA,EAAM,aAAa,qBAAqB,EACjD,OAASA,EAAM,aAAa,sBAAsB,EAClD,QAASA,EAAM,aAAa,uBAAuB,EACnD,OAASA,EAAM,aAAa,sBAAsB,CACpD,CACF,EAAa,EACbO,EAAY,CAAC,CACf,CACJ,CASO,SAASC,GACdC,EACgC,CAChC,IAAMX,EAASY,EAA8B,QAASD,CAAE,EAClDE,EAAOC,EAAE,OAAQ,CAAE,KAAM,aAAc,CAAC,EAC9C,SAAS,KAAK,YAAYD,CAAI,EAG9B,IAAME,EAASD,EAAE,OAAQ,CAAE,KAAM,cAAe,CAAC,EACjD,SAAS,KAAK,YAAYC,CAAM,EAGhC,IAAMC,EAASC,GAAW,+BAA+B,EACzD,OAAOC,EAAM,IAAM,CACjB,IAAMC,EAAQ,IAAIC,EAClB,OAAAD,EAAM,UAAUE,GAAW,CAIzB,GAHA,SAAS,KAAK,aAAa,0BAA2B,EAAE,EAGpDA,EAAQ,MAAM,QAAU,yBAA0B,CACpD,IAAMC,EAAQ,WAAW,+BAA+B,EAClDpB,EAAQ,SAAS,cAAcoB,EAAM,QACvC,wDACA,sDACJ,EAGAD,EAAQ,MAAM,OAAUnB,EAAM,aAAa,sBAAsB,EACjEmB,EAAQ,MAAM,QAAUnB,EAAM,aAAa,uBAAuB,EAClEmB,EAAQ,MAAM,OAAUnB,EAAM,aAAa,sBAAsB,CACnE,CAGA,OAAW,CAACqB,EAAKC,CAAK,IAAK,OAAO,QAAQH,EAAQ,KAAK,EACrD,SAAS,KAAK,aAAa,iBAAiBE,CAAG,GAAIC,CAAK,EAG1D,QAASrB,EAAQ,EAAGA,EAAQH,EAAO,OAAQG,IAAS,CAClD,IAAMsB,EAAQzB,EAAOG,CAAK,EAAE,mBACxBsB,aAAiB,cACnBA,EAAM,OAASJ,EAAQ,QAAUlB,EACrC,CAGA,SAAS,YAAakB,CAAO,CAC/B,CAAC,EAGDF,EACG,KACCZ,EAAI,IAAM,CACR,IAAMmB,EAASC,GAAoB,QAAQ,EACrCC,EAAS,OAAO,iBAAiBF,CAAM,EAG7C,OAAAX,EAAO,QAAUa,EAAM,YAGhBA,EAAM,gBAAgB,MAAM,MAAM,EACtC,IAAIJ,IAAU,CAACA,GAAO,SAAS,EAAE,EAAE,SAAS,EAAG,GAAG,CAAC,EACnD,KAAK,EAAE,CACZ,CAAC,CACH,EACG,UAAUK,GAAShB,EAAK,QAAU,IAAIgB,CAAK,EAAE,EAGlDV,EAAM,KAAKW,GAAUC,EAAc,CAAC,EACjC,UAAU,IAAM,CACf,SAAS,KAAK,gBAAgB,yBAAyB,CACzD,CAAC,EAGIhC,GAAaC,CAAM,EACvB,KACCgC,EAAUhB,EAAO,KAAKiB,GAAK,CAAC,CAAC,CAAC,EAC9BC,GAAO,EACPC,EAAIC,GAASjB,EAAM,KAAKiB,CAAK,CAAC,EAC9BC,EAAS,IAAMlB,EAAM,SAAS,CAAC,EAC/BZ,EAAI6B,GAAUE,EAAA,CAAE,IAAK3B,GAAOyB,EAAQ,CACtC,CACJ,CAAC,CACH,CCnIO,SAASG,GACdC,EAAiB,CAAE,UAAAC,CAAU,EACI,CAGjC,OAAOC,EAAM,IAAM,CACjB,IAAMC,EAAQ,IAAIC,EAClB,OAAAD,EAAM,UAAU,CAAC,CAAE,MAAAE,CAAM,IAAM,CAC7BL,EAAG,MAAM,YAAY,sBAAuB,GAAGK,CAAK,EAAE,CACxD,CAAC,EAGMJ,EACJ,KACCK,EAAID,GAASF,EAAM,KAAK,CAAE,MAAAE,CAAM,CAAC,CAAC,EAClCE,EAAS,IAAMJ,EAAM,SAAS,CAAC,EAC/BK,EAAIH,IAAU,CAAE,IAAKL,EAAI,MAAAK,CAAM,EAAE,CACnC,CACJ,CAAC,CACH,CChEA,IAAAI,GAAwB,SAiCxB,SAASC,GAAQC,EAAyB,CACxCA,EAAG,aAAa,kBAAmB,EAAE,EACrC,IAAMC,EAAOD,EAAG,QAAQ,aAAa,EAC/BE,EAAOD,EACTA,EAAK,aAAa,WAAW,EAC7BD,EAAG,UACP,OAAAA,EAAG,gBAAgB,iBAAiB,EAC7BE,EAAK,QAAQ,CACtB,CAWO,SAASC,GACd,CAAE,OAAAC,CAAO,EACH,CACF,GAAAC,QAAY,YAAY,GAC1B,IAAIC,EAA8BC,GAAc,CAC9C,IAAI,GAAAF,QAAY,iDAAkD,CAChE,KAAML,GACJA,EAAG,aAAa,qBAAqB,GACrCD,GAAQS,EACNR,EAAG,aAAa,uBAAuB,CACzC,CAAC,CAEL,CAAC,EACE,GAAG,UAAWS,GAAMF,EAAW,KAAKE,CAAE,CAAC,CAC5C,CAAC,EACE,KACCC,EAAID,GAAM,CACQA,EAAG,QACX,MAAM,CAChB,CAAC,EACDE,EAAI,IAAMC,GAAY,kBAAkB,CAAC,CAC3C,EACG,UAAUR,CAAM,CAEzB,CCrCA,SAASS,GAAQC,EAAUC,EAAW,CACpC,OAAAD,EAAI,SAAWC,EAAK,SACpBD,EAAI,SAAWC,EAAK,SACbD,CACT,CA2BA,SAASE,GAAQC,EAAoBF,EAAoB,CACvD,IAAMG,EAAmB,IAAI,IAC7B,QAAWC,KAAMC,EAAY,MAAOH,CAAQ,EAAG,CAC7C,IAAMH,EAAMO,EAAW,MAAOF,CAAE,EAG1BG,EAAQ,CAACT,GAAQ,IAAI,IAAIC,EAAI,WAAY,EAAGC,CAAI,CAAC,EACvDG,EAAQ,IAAI,GAAGI,EAAM,CAAC,CAAC,GAAIA,CAAK,EAGhC,QAAWC,KAAQH,EAAY,kBAAmBD,CAAE,EAAG,CACrD,IAAMK,EAAOD,EAAK,aAAa,MAAM,EACjCC,GAAQ,MACVF,EAAM,KAAKT,GAAQ,IAAI,IAAIW,CAAI,EAAGT,CAAI,CAAC,CAC3C,CACF,CAGA,OAAOG,CACT,CAgBO,SAASO,GAAaV,EAAyC,CACpE,OAAOW,GAAW,IAAI,IAAI,cAAeX,CAAI,CAAC,EAC3C,KACCY,EAAIV,GAAYD,GAAQC,EAAU,IAAI,IAAIF,CAAI,CAAC,CAAC,EAChDa,GAAW,IAAMC,EAAG,IAAI,GAAK,CAAC,CAChC,CACJ,CClDA,SAASC,GACPC,EAAgBC,EACC,CACjB,GAAI,EAAED,EAAG,kBAAkB,SACzB,OAAOE,EAIT,IAAMC,EAAKH,EAAG,OAAO,QAAQ,GAAG,EAChC,GAAIG,IAAO,KACT,OAAOD,EAMT,GAAIC,EAAG,QAAUH,EAAG,SAAWA,EAAG,QAChC,OAAOE,EAQT,IAAME,EAAM,IAAI,IAAID,EAAG,IAAI,EAO3B,OANAC,EAAI,OAASA,EAAI,KAAO,GAMnBH,EAAQ,IAAI,GAAGG,CAAG,EAAE,GASzBJ,EAAG,eAAe,EACXK,EAAG,IAAI,IAAIF,EAAG,IAAI,CAAC,GATjBD,CAUX,CASA,SAASI,GAAKC,EAA8C,CAC1D,IAAMC,EAAO,IAAI,IACjB,QAAWL,KAAMM,EAAY,aAAcF,EAAS,IAAI,EACtDC,EAAK,IAAIL,EAAG,UAAWA,CAAE,EAG3B,OAAOK,CACT,CAYA,SAASE,GAAQH,EAA0C,CACzD,QAAWJ,KAAMM,EAAY,gBAAiBF,CAAQ,EACpD,QAAWI,IAAO,CAAC,OAAQ,KAAK,EAAG,CACjC,IAAMC,EAAQT,EAAG,aAAaQ,CAAG,EACjC,GAAIC,GAAS,CAAC,qBAAqB,KAAKA,CAAK,EAAG,CAE9CT,EAAGQ,CAAG,EAAIR,EAAGQ,CAAG,EAChB,KACF,CACF,CAGF,OAAON,EAAGE,CAAQ,CACpB,CASA,SAASM,GAAOC,EAAsC,CACpD,QAAWC,IAAY,CACrB,+BACA,gCACA,mCACA,+BACA,2BACA,2BACA,GAAGC,EAAQ,wBAAwB,EAC/B,CAAC,0BAA0B,EAC3B,CAAC,CACP,EAAG,CACD,IAAMC,EAASC,GAAmBH,CAAQ,EACpCI,EAASD,GAAmBH,EAAUD,CAAI,EAE9C,OAAOG,GAAW,aAClB,OAAOE,GAAW,aAElBF,EAAO,YAAYE,CAAM,CAE7B,CAGA,IAAMX,EAAOF,GAAK,QAAQ,EAC1B,OAAW,CAACc,EAAMjB,CAAE,IAAKG,GAAKQ,CAAI,EAC5BN,EAAK,IAAIY,CAAI,EACfZ,EAAK,OAAOY,CAAI,EAEhB,SAAS,KAAK,YAAYjB,CAAE,EAGhC,QAAWA,KAAMK,EAAK,OAAO,EAAG,CAC9B,IAAMa,EAAOlB,EAAG,aAAa,MAAM,EAI/BkB,IAAS,eAAiBA,IAAS,gBACrClB,EAAG,OAAO,CACd,CAIA,IAAMmB,EAAYC,GAAoB,WAAW,EACjD,OAAOC,GAAOf,EAAY,SAAUa,CAAS,CAAC,EAC3C,KACCG,EAAUtB,GAAM,CACd,IAAMuB,EAASZ,EAAK,cAAc,QAAQ,EAC1C,GAAIX,EAAG,IAAK,CACV,QAAWkB,KAAQlB,EAAG,kBAAkB,EACtCuB,EAAO,aAAaL,EAAMlB,EAAG,aAAakB,CAAI,CAAE,EAClD,OAAAlB,EAAG,YAAYuB,CAAM,EAGd,IAAIC,EAAWC,GAAY,CAChCF,EAAO,OAAS,IAAME,EAAS,SAAS,CAC1C,CAAC,CAGH,KACE,QAAAF,EAAO,YAAcvB,EAAG,YACxBA,EAAG,YAAYuB,CAAM,EACdxB,CAEX,CAAC,EACD2B,GAAe,EACfC,GAAQ,QAAQ,CAClB,CACJ,CAgBO,SAASC,GACd,CAAE,UAAAC,EAAW,UAAAC,EAAW,UAAAC,CAAU,EACZ,CACtB,IAAMC,EAASC,GAAc,EAC7B,GAAI,SAAS,WAAa,QACxB,OAAOlC,EAIT,IAAMmC,EAAWC,GAAaH,EAAO,IAAI,EAUzC9B,EAAG,QAAQ,EACR,UAAUK,EAAO,EAUpB,IAAM6B,EACJC,EAAsB,SAAS,KAAM,OAAO,EACzC,KACCC,GAAkBJ,CAAQ,EAC1BZ,EAAU,CAAC,CAACzB,EAAIC,CAAO,IAAMF,GAAOC,EAAIC,CAAO,CAAC,EAChDyC,GAAM,CACR,EAIEC,EACJH,EAAyB,OAAQ,UAAU,EACxC,KACCI,EAAIC,EAAW,EACfH,GAAM,CACR,EAMJH,EAAS,KAAKO,GAAeb,CAAS,CAAC,EACpC,UAAU,CAAC,CAAC7B,EAAK,CAAE,OAAA2C,CAAO,CAAC,IAAM,CAChC,QAAQ,aAAaA,EAAQ,EAAE,EAC/B,QAAQ,UAAU,KAAM,GAAI3C,CAAG,CACjC,CAAC,EAMH4C,EAAMT,EAAUI,CAAQ,EACrB,UAAUX,CAAS,EActB,IAAMiB,EACJjB,EAAU,KACRkB,EAAwB,UAAU,EAClCzB,EAAUrB,GAAO+C,GAAY/C,EAAK,CAAE,UAAA8B,CAAU,CAAC,EAC5C,KACCkB,GAAW,KACTC,GAAYjD,EAAK,EAAI,EACdF,EACR,CACH,CACF,EAIAuB,EAAUf,EAAO,EACjBe,EAAUZ,EAAM,EAChB6B,GAAM,CACR,EAUF,OAAAM,EACEC,EAAU,KAAKH,GAAed,EAAW,CAACsB,EAAGlD,IAAQA,CAAG,CAAC,EASzD4B,EAAU,KACRkB,EAAwB,UAAU,EAClCzB,EAAU,IAAMO,CAAS,EACzBkB,EAAwB,MAAM,CAChC,EAQAlB,EAAU,KACRuB,EAAqB,CAACC,EAAGC,IACvBD,EAAE,WAAaC,EAAE,UACjBD,EAAE,OAAaC,EAAE,IAClB,EACDhC,EAAU,IAAMc,CAAQ,EACxBmB,EAAI,IAAM,QAAQ,KAAK,CAAC,CAC1B,CACF,EACG,UAAUtD,GAAO,CAzYtB,IAAAuD,EAAAC,EA+YU,QAAQ,QAAU,MAAQ,CAACxD,EAAI,KACjC,OAAO,SAAS,GAAGwD,GAAAD,EAAA,QAAQ,QAAR,YAAAA,EAAe,IAAf,KAAAC,EAAoB,CAAC,GAExC,QAAQ,kBAAoB,OAC5BC,GAAgBzD,EAAI,IAAI,EACxB,QAAQ,kBAAoB,SAEhC,CAAC,EAMH4B,EAAU,UAAU,IAAM,CACxB,QAAQ,kBAAoB,QAC9B,CAAC,EAMDQ,EAAU,OAAQ,cAAc,EAC7B,UAAU,IAAM,CACf,QAAQ,kBAAoB,MAC9B,CAAC,EAMHP,EAAU,KACRiB,EAAwB,QAAQ,EAChCY,GAAa,GAAG,CAClB,EACG,UAAU,CAAC,CAAE,OAAAf,CAAO,IAAM,CACzB,QAAQ,aAAaA,EAAQ,EAAE,CACjC,CAAC,EAGIE,CACT,CCjaA,IAAAc,GAAuB,SAqChB,SAASC,GACdC,EAC0B,CAE1B,IAAMC,EAAQD,EAAO,UAAU,MAAM,GAAG,EAAE,IAAIE,GAC/BA,EAAK,QAAQ,sBAAuB,EAAE,EACvC,SAAW,EAAI,SAAMA,CAClC,EACE,KAAK,GAAG,EAELC,EAAY,IAAI,OAAOF,EAAO,KAAK,EACnCG,EAAY,CAACC,EAAYC,EAAcJ,IACpC,GAAGI,CAAI,2BAA2BJ,CAAI,UAI/C,OAAQK,GAAkB,CACxBA,EAAQA,EACL,QAAQ,gBAAiB,GAAG,EAC5B,KAAK,EAGR,IAAMC,EAAQ,IAAI,OAAO,MAAMR,EAAO,SAAS,MAC7CO,EACG,QAAQ,uBAAwB,MAAM,EACtC,QAAQJ,EAAW,GAAG,CAC3B,IAAK,KAAK,EAGV,OAAOM,MAAS,GAAAC,SAAWD,CAAK,EAC7B,QAAQD,EAAOJ,CAAS,EACxB,QAAQ,8BAA+B,IAAI,CAChD,CACF,CCEO,SAASO,GACdC,EAC+B,CAC/B,OAAOA,EAAQ,OAAS,CAC1B,CASO,SAASC,GACdD,EACgC,CAChC,OAAOA,EAAQ,OAAS,CAC1B,CC1CO,SAASE,GACdC,EAAaC,EACW,CACxB,IAAMC,EAAUC,GAA2BH,CAAG,EAC9C,OAAAI,EACEC,EAAG,SAAS,WAAa,OAAO,EAChCC,GAAY,QAAQ,CACtB,EACG,KACCC,GAAMC,GAAUA,CAAM,EACtBC,EAAU,IAAMR,CAAM,CACxB,EACG,UAAU,CAAC,CAAE,OAAAS,EAAQ,KAAAC,CAAK,IAAMT,EAAQ,KAAK,CAC5C,OACA,KAAM,CACJ,OAAAQ,EACA,KAAAC,EACA,QAAS,CACP,QAASC,EAAQ,gBAAgB,CACnC,CACF,CACF,CAAC,CAAC,EAGCV,CACT,CCxBO,SAASW,GACd,CAAE,UAAAC,CAAU,EACN,CACN,IAAMC,EAASC,GAAc,EACvBC,EAAYC,GAChB,IAAI,IAAI,mBAAoBH,EAAO,IAAI,CACzC,EACG,KACCI,GAAW,IAAMC,CAAK,CACxB,EAGIC,EAAWJ,EACd,KACCK,EAAIC,GAAY,CACd,GAAM,CAAC,CAAEC,CAAO,EAAIT,EAAO,KAAK,MAAM,aAAa,EACnD,OAAOQ,EAAS,KAAK,CAAC,CAAE,QAAAE,EAAS,QAAAC,CAAQ,IACvCD,IAAYD,GAAWE,EAAQ,SAASF,CAAO,CAChD,GAAKD,EAAS,CAAC,CAClB,CAAC,CACH,EAGFN,EACG,KACCK,EAAIC,GAAY,IAAI,IAAIA,EAAS,IAAIE,GAAW,CAC9C,GAAG,IAAI,IAAI,MAAMA,EAAQ,OAAO,IAAKV,EAAO,IAAI,CAAC,GACjDU,CACF,CAAC,CAAC,CAAC,EACHE,EAAUC,GAAQC,EAAsB,SAAS,KAAM,OAAO,EAC3D,KACCC,EAAOC,GAAM,CAACA,EAAG,SAAW,CAACA,EAAG,OAAO,EACvCC,GAAeX,CAAQ,EACvBM,EAAU,CAAC,CAACI,EAAIP,CAAO,IAAM,CAC3B,GAAIO,EAAG,kBAAkB,QAAS,CAChC,IAAME,EAAKF,EAAG,OAAO,QAAQ,GAAG,EAChC,GAAIE,GAAM,CAACA,EAAG,QAAUL,EAAK,IAAIK,EAAG,IAAI,EAAG,CACzC,IAAMC,EAAMD,EAAG,KAWf,MAAI,CAACF,EAAG,OAAO,QAAQ,aAAa,GAClBH,EAAK,IAAIM,CAAG,IACZV,EACPJ,GAEXW,EAAG,eAAe,EACXI,EAAGD,CAAG,EACf,CACF,CACA,OAAOd,CACT,CAAC,EACDO,EAAUO,GAAO,CACf,GAAM,CAAE,QAAAT,CAAQ,EAAIG,EAAK,IAAIM,CAAG,EAChC,OAAOE,GAAa,IAAI,IAAIF,CAAG,CAAC,EAC7B,KACCZ,EAAIe,GAAW,CAEb,IAAMC,EADWC,GAAY,EACP,KAAK,QAAQxB,EAAO,KAAM,EAAE,EAClD,OAAOsB,EAAQ,IAAIC,EAAK,MAAM,GAAG,EAAE,CAAC,CAAC,EACjC,IAAI,IAAI,MAAMb,CAAO,IAAIa,CAAI,GAAIvB,EAAO,IAAI,EAC5C,IAAI,IAAImB,CAAG,CACjB,CAAC,CACH,CACJ,CAAC,CACH,CACF,CACF,EACG,UAAUA,GAAOM,GAAYN,EAAK,EAAI,CAAC,EAG5CO,EAAc,CAACxB,EAAWI,CAAQ,CAAC,EAChC,UAAU,CAAC,CAACE,EAAUC,CAAO,IAAM,CACpBkB,EAAW,mBAAmB,EACtC,YAAYC,GAAsBpB,EAAUC,CAAO,CAAC,CAC5D,CAAC,EAGHV,EAAU,KAAKa,EAAU,IAAMN,CAAQ,CAAC,EACrC,UAAUG,GAAW,CA5J1B,IAAAoB,EA+JM,IAAIC,EAAW,SAAS,aAAc,cAAc,EACpD,GAAIA,IAAa,KAAM,CACrBA,EAAW,GAGX,IAAIC,IAAUF,EAAA7B,EAAO,UAAP,YAAA6B,EAAgB,UAAW,SACpC,MAAM,QAAQE,CAAO,IACxBA,EAAU,CAACA,CAAO,GAGpBC,EAAM,QAAWC,KAAUF,EACzB,QAAWrB,KAAWD,EAAQ,QAAQ,OAAOA,EAAQ,OAAO,EAC1D,GAAI,IAAI,OAAOwB,EAAQ,GAAG,EAAE,KAAKvB,CAAO,EAAG,CACzCoB,EAAW,GACX,MAAME,CACR,CAGJ,SAAS,aAAcF,EAAU,cAAc,CACjD,CAGA,GAAIA,EACF,QAAWI,KAAWC,GAAqB,UAAU,EACnDD,EAAQ,OAAS,EACvB,CAAC,CACL,CCrFO,SAASE,GACdC,EAAsB,CAAE,QAAAC,CAAQ,EACP,CAGzB,GAAM,CAAE,aAAAC,CAAa,EAAIC,GAAY,EACjCD,EAAa,IAAI,GAAG,IACtBE,GAAU,SAAU,EAAI,EAGxBJ,EAAG,MAAQE,EAAa,IAAI,GAAG,EAC/BF,EAAG,MAAM,EAGTK,GAAY,QAAQ,EACjB,KACCC,GAAMC,GAAU,CAACA,CAAM,CACzB,EACG,UAAU,IAAM,CACf,IAAMC,EAAML,GAAY,EACxBK,EAAI,aAAa,OAAO,GAAG,EAC3B,QAAQ,aAAa,CAAC,EAAG,GAAI,GAAGA,CAAG,EAAE,CACvC,CAAC,GAIP,IAAMC,EAASC,GAAkBV,CAAE,EAC7BW,EAASC,EACbX,EAAQ,KAAKK,GAAMO,EAAoB,CAAC,EACxCC,EAAUd,EAAI,OAAO,EACrBS,CACF,EACG,KACCM,EAAI,IAAMf,EAAG,KAAK,EAClBgB,EAAqB,CACvB,EAGF,OAAOC,EAAc,CAACN,EAAQF,CAAM,CAAC,EAClC,KACCM,EAAI,CAAC,CAACG,EAAOC,CAAK,KAAO,CAAE,MAAAD,EAAO,MAAAC,CAAM,EAAE,EAC1CC,EAAY,CAAC,CACf,CACJ,CAUO,SAASC,GACdrB,EAAsB,CAAE,QAAAC,CAAQ,EACsB,CACtD,IAAMqB,EAAQ,IAAIC,EACZC,EAAQF,EAAM,KAAKG,GAAe,EAAGC,GAAQ,EAAI,CAAC,EAGxDT,EAAc,CACZhB,EAAQ,KAAKK,GAAMO,EAAoB,CAAC,EACxCS,CACF,EAAG,CAACK,EAAGC,IAAUA,CAAK,EACnB,KACCC,EAAwB,OAAO,CACjC,EACG,UAAU,CAAC,CAAE,MAAAX,CAAM,IAAMjB,EAAQ,KAAK,CACrC,OACA,KAAMiB,CACR,CAAC,CAAC,EAGNI,EACG,KACCO,EAAwB,OAAO,CACjC,EACG,UAAU,CAAC,CAAE,MAAAV,CAAM,IAAM,CACpBA,GACFf,GAAU,SAAUe,CAAK,CAC7B,CAAC,EAGLL,EAAUd,EAAG,KAAO,OAAO,EACxB,KACC8B,EAAUN,CAAK,CACjB,EACG,UAAU,IAAMxB,EAAG,MAAM,CAAC,EAM/B,IAAM+B,EAAQC,EAAW,uBAAuB,EAChD,OAAAlB,EAAUiB,EAAO,OAAO,EACrB,UAAU,IAAM/B,EAAG,MAAM,CAAC,EAGtBD,GAAiBC,EAAI,CAAE,QAAAC,CAAQ,CAAC,EACpC,KACCgC,EAAIC,GAASZ,EAAM,KAAKY,CAAK,CAAC,EAC9BC,EAAS,IAAMb,EAAM,SAAS,CAAC,EAC/BP,EAAImB,GAAUE,EAAA,CAAE,IAAKpC,GAAOkC,EAAQ,EACpCd,EAAY,CAAC,CACf,CACJ,CCnHO,SAASiB,GACdC,EAAiB,CAAE,QAAAC,EAAS,OAAAC,CAAO,EACE,CACrC,IAAMC,EAAQ,IAAIC,EACZC,EAAYC,GAAqBN,EAAG,aAAc,EACrD,KACCO,EAAO,OAAO,CAChB,EAGIC,EAAYR,EAAG,cAGfS,EAAOC,EAAW,wBAAyBV,CAAE,EAC7CW,EAAOD,EAAW,uBAAwBV,CAAE,EAGlDY,GAAY,QAAQ,EACjB,UAAUC,GAAUF,EAAK,aACxB,OAAQE,EAAS,OAAS,cAC5B,CAAC,EAGHV,EACG,KACCW,GAAeZ,CAAM,EACrBa,GAAUd,EAAQ,KAAKe,GAAMC,EAAoB,CAAC,CAAC,CACrD,EACG,UAAU,CAAC,CAAC,CAAE,MAAAC,CAAM,EAAG,CAAE,MAAAC,CAAM,CAAC,IAAM,CACrC,OAAQD,EAAM,OAAQ,CAGpB,IAAK,GACHT,EAAK,YAAcU,EAAM,OACrBC,GAAY,oBAAoB,EAChCA,GAAY,2BAA2B,EAC3C,MAGF,IAAK,GACHX,EAAK,YAAcW,GAAY,mBAAmB,EAClD,MAGF,QACE,IAAMC,EAAQC,GAAMJ,EAAM,MAAM,EAChCT,EAAK,YAAcW,GAAY,sBAAuBC,CAAK,CAC/D,CACF,CAAC,EAGL,IAAME,EAAUpB,EACb,KACCqB,EAAI,IAAMb,EAAK,UAAY,EAAE,EAC7Bc,EAAU,CAAC,CAAE,MAAAP,CAAM,IAAMQ,EACvBC,EAAG,GAAGT,EAAM,MAAM,EAAG,EAAE,CAAC,EACxBS,EAAG,GAAGT,EAAM,MAAM,EAAE,CAAC,EAClB,KACCU,GAAY,CAAC,EACbC,GAAQxB,CAAS,EACjBoB,EAAU,CAAC,CAACK,CAAK,IAAMA,CAAK,CAC9B,CACJ,CAAC,EACDC,EAAIC,EAAsB,EAC1BC,GAAM,CACR,EAGF,OAAAV,EAAQ,UAAUW,GAAQvB,EAAK,YAAYuB,CAAI,CAAC,EAChDX,EACG,KACCY,GAASD,GAAQ,CACf,IAAME,EAAUC,GAAmB,UAAWH,CAAI,EAClD,OAAI,OAAOE,GAAY,YACdE,EAGFC,EAAUH,EAAS,QAAQ,EAC/B,KACCI,EAAUrC,CAAK,EACf4B,EAAI,IAAMK,CAAO,CACnB,CACJ,CAAC,CACH,EACG,UAAUA,GAAW,CAElBA,EAAQ,OAAS,IACjBA,EAAQ,WAAa5B,EAAU,WAE/BA,EAAU,SAAS,CAAE,IAAK4B,EAAQ,SAAU,CAAC,CACjD,CAAC,EAGWnC,EACb,KACCM,EAAOkC,EAAqB,EAC5BV,EAAI,CAAC,CAAE,KAAAW,CAAK,IAAMA,CAAI,CACxB,EAIC,KACClB,EAAImB,GAASxC,EAAM,KAAKwC,CAAK,CAAC,EAC9BC,EAAS,IAAMzC,EAAM,SAAS,CAAC,EAC/B4B,EAAIY,GAAUE,EAAA,CAAE,IAAK7C,GAAO2C,EAAQ,CACtC,CACJ,CCpHO,SAASG,GACdC,EAAkB,CAAE,OAAAC,CAAO,EACF,CACzB,OAAOA,EACJ,KACCC,EAAI,CAAC,CAAE,MAAAC,CAAM,IAAM,CACjB,IAAMC,EAAMC,GAAY,EACxB,OAAAD,EAAI,KAAO,GAGXD,EAAQA,EACL,QAAQ,OAAQ,GAAG,EACnB,QAAQ,KAAM,KAAK,EACnB,QAAQ,KAAM,KAAK,EAGtBC,EAAI,OAAS,KAAKD,CAAK,GAChB,CAAE,IAAAC,CAAI,CACf,CAAC,CACH,CACJ,CAUO,SAASE,GACdC,EAAuBC,EACa,CACpC,IAAMC,EAAQ,IAAIC,EACZC,EAAQF,EAAM,KAAKG,GAAe,EAAGC,GAAQ,EAAI,CAAC,EACxD,OAAAJ,EAAM,UAAU,CAAC,CAAE,IAAAL,CAAI,IAAM,CAC3BG,EAAG,aAAa,sBAAuBA,EAAG,IAAI,EAC9CA,EAAG,KAAO,GAAGH,CAAG,EAClB,CAAC,EAGDU,EAAUP,EAAI,OAAO,EAClB,KACCQ,EAAUJ,CAAK,CACjB,EACG,UAAUK,GAAMA,EAAG,eAAe,CAAC,EAGjCjB,GAAiBQ,EAAIC,CAAO,EAChC,KACCS,EAAIC,GAAST,EAAM,KAAKS,CAAK,CAAC,EAC9BC,EAAS,IAAMV,EAAM,SAAS,CAAC,EAC/BP,EAAIgB,GAAUE,EAAA,CAAE,IAAKb,GAAOW,EAAQ,CACtC,CACJ,CCpDO,SAASG,GACdC,EAAiB,CAAE,QAAAC,EAAS,UAAAC,CAAU,EACA,CACtC,IAAMC,EAAQ,IAAIC,EAGZC,EAASC,GAAoB,cAAc,EAC3CC,EAASC,EACbC,EAAUJ,EAAO,SAAS,EAC1BI,EAAUJ,EAAO,OAAO,CAC1B,EACG,KACCK,GAAUC,EAAc,EACxBC,EAAI,IAAMP,EAAM,KAAK,EACrBQ,EAAqB,CACvB,EAGF,OAAAV,EACG,KACCW,GAAkBP,CAAM,EACxBK,EAAI,CAAC,CAAC,CAAE,QAAAG,CAAQ,EAAGC,CAAK,IAAM,CAC5B,IAAMC,EAAQD,EAAM,MAAM,UAAU,EACpC,GAAID,GAAA,MAAAA,EAAS,QAAUE,EAAMA,EAAM,OAAS,CAAC,EAAG,CAC9C,IAAMC,EAAOH,EAAQA,EAAQ,OAAS,CAAC,EACnCG,EAAK,WAAWD,EAAMA,EAAM,OAAS,CAAC,CAAC,IACzCA,EAAMA,EAAM,OAAS,CAAC,EAAIC,EAC9B,MACED,EAAM,OAAS,EAEjB,OAAOA,CACT,CAAC,CACH,EACG,UAAUA,GAASjB,EAAG,UAAYiB,EAChC,KAAK,EAAE,EACP,QAAQ,MAAO,QAAQ,CAC1B,EAGJf,EACG,KACCiB,EAAO,CAAC,CAAE,KAAAC,CAAK,IAAMA,IAAS,QAAQ,CACxC,EACG,UAAUC,GAAO,CAChB,OAAQA,EAAI,KAAM,CAGhB,IAAK,aAEDrB,EAAG,UAAU,QACbK,EAAM,iBAAmBA,EAAM,MAAM,SAErCA,EAAM,MAAQL,EAAG,WACnB,KACJ,CACF,CAAC,EAGWC,EACb,KACCkB,EAAOG,EAAqB,EAC5BV,EAAI,CAAC,CAAE,KAAAW,CAAK,IAAMA,CAAI,CACxB,EAIC,KACCC,EAAIC,GAAStB,EAAM,KAAKsB,CAAK,CAAC,EAC9BC,EAAS,IAAMvB,EAAM,SAAS,CAAC,EAC/BS,EAAI,KAAO,CAAE,IAAKZ,CAAG,EAAE,CACzB,CACJ,CCjDO,SAAS2B,GACdC,EAAiB,CAAE,OAAAC,EAAQ,UAAAC,CAAU,EACN,CAC/B,IAAMC,EAASC,GAAc,EAC7B,GAAI,CACF,IAAMC,EAAUC,GAAkBH,EAAO,OAAQF,CAAM,EAGjDM,EAASC,GAAoB,eAAgBR,CAAE,EAC/CS,EAASD,GAAoB,gBAAiBR,CAAE,EAGtDU,EAAwBV,EAAI,OAAO,EAChC,KACCW,EAAO,CAAC,CAAE,OAAAC,CAAO,IACfA,aAAkB,SAAW,CAAC,CAACA,EAAO,QAAQ,GAAG,CAClD,CACH,EACG,UAAU,IAAMC,GAAU,SAAU,EAAK,CAAC,EAG/CX,EACG,KACCS,EAAO,CAAC,CAAE,KAAAG,CAAK,IAAMA,IAAS,QAAQ,CACxC,EACG,UAAUC,GAAO,CAChB,IAAMC,EAASC,GAAiB,EAChC,OAAQF,EAAI,KAAM,CAGhB,IAAK,QACH,GAAIC,IAAWT,EAAO,CACpB,IAAMW,EAAU,IAAI,IACpB,QAAWC,KAAUC,EACnB,sBAAuBX,CACzB,EAAG,CACD,IAAMY,EAAUF,EAAO,kBACvBD,EAAQ,IAAIC,EAAQ,WAClBE,EAAQ,aAAa,eAAe,CACtC,CAAC,CACH,CAGA,GAAIH,EAAQ,KAAM,CAChB,GAAM,CAAC,CAACI,CAAI,CAAC,EAAI,CAAC,GAAGJ,CAAO,EAAE,KAAK,CAAC,CAAC,CAAEK,CAAC,EAAG,CAAC,CAAEC,CAAC,IAAMA,EAAID,CAAC,EAC1DD,EAAK,MAAM,CACb,CAGAP,EAAI,MAAM,CACZ,CACA,MAGF,IAAK,SACL,IAAK,MACHF,GAAU,SAAU,EAAK,EACzBN,EAAM,KAAK,EACX,MAGF,IAAK,UACL,IAAK,YACH,GAAI,OAAOS,GAAW,YACpBT,EAAM,MAAM,MACP,CACL,IAAMkB,EAAM,CAAClB,EAAO,GAAGa,EACrB,wDACAX,CACF,CAAC,EACKiB,EAAI,KAAK,IAAI,GACjB,KAAK,IAAI,EAAGD,EAAI,QAAQT,CAAM,CAAC,EAAIS,EAAI,QACrCV,EAAI,OAAS,UAAY,GAAK,IAE9BU,EAAI,MAAM,EACdA,EAAIC,CAAC,EAAE,MAAM,CACf,CAGAX,EAAI,MAAM,EACV,MAGF,QACMR,IAAUU,GAAiB,GAC7BV,EAAM,MAAM,CAClB,CACF,CAAC,EAGLL,EACG,KACCS,EAAO,CAAC,CAAE,KAAAG,CAAK,IAAMA,IAAS,QAAQ,CACxC,EACG,UAAUC,GAAO,CAChB,OAAQA,EAAI,KAAM,CAGhB,IAAK,IACL,IAAK,IACL,IAAK,IACHR,EAAM,MAAM,EACZA,EAAM,OAAO,EAGbQ,EAAI,MAAM,EACV,KACJ,CACF,CAAC,EAGL,IAAMY,EAASC,GAAiBrB,EAAO,CAAE,QAAAF,CAAQ,CAAC,EAClD,OAAOwB,EACLF,EACAG,GAAkBrB,EAAQ,CAAE,QAAAJ,EAAS,OAAAsB,CAAO,CAAC,CAC/C,EACG,KACCI,GAGE,GAAGC,GAAqB,eAAgBhC,CAAE,EACvC,IAAIiC,GAASC,GAAiBD,EAAO,CAAE,OAAAN,CAAO,CAAC,CAAC,EAGnD,GAAGK,GAAqB,iBAAkBhC,CAAE,EACzC,IAAIiC,GAASE,GAAmBF,EAAO,CAAE,QAAA5B,EAAS,UAAAH,CAAU,CAAC,CAAC,CACnE,CACF,CAGJ,OAASkC,EAAK,CACZ,OAAApC,EAAG,OAAS,GACLqC,EACT,CACF,CCnKO,SAASC,GACdC,EAAiB,CAAE,OAAAC,EAAQ,UAAAC,CAAU,EACG,CACxC,OAAOC,EAAc,CACnBF,EACAC,EACG,KACCE,EAAUC,GAAY,CAAC,EACvBC,EAAOC,GAAO,CAAC,CAACA,EAAI,aAAa,IAAI,GAAG,CAAC,CAC3C,CACJ,CAAC,EACE,KACCC,EAAI,CAAC,CAACC,EAAOF,CAAG,IAAMG,GAAuBD,EAAM,MAAM,EACvDF,EAAI,aAAa,IAAI,GAAG,CAC1B,CAAC,EACDC,EAAIG,GAAM,CA1FhB,IAAAC,EA2FQ,IAAMC,EAAQ,IAAI,IAGZC,EAAK,SAAS,mBAAmBd,EAAI,WAAW,SAAS,EAC/D,QAASe,EAAOD,EAAG,SAAS,EAAGC,EAAMA,EAAOD,EAAG,SAAS,EACtD,IAAIF,EAAAG,EAAK,gBAAL,MAAAH,EAAoB,aAAc,CACpC,IAAMI,EAAWD,EAAK,YAChBE,EAAWN,EAAGK,CAAQ,EACxBC,EAAS,OAASD,EAAS,QAC7BH,EAAM,IAAIE,EAAmBE,CAAQ,CACzC,CAIF,OAAW,CAACF,EAAMG,CAAI,IAAKL,EAAO,CAChC,GAAM,CAAE,WAAAM,CAAW,EAAIC,EAAE,OAAQ,KAAMF,CAAI,EAC3CH,EAAK,YAAY,GAAG,MAAM,KAAKI,CAAU,CAAC,CAC5C,CAGA,MAAO,CAAE,IAAKnB,EAAI,MAAAa,CAAM,CAC1B,CAAC,CACH,CACJ,CCPO,SAASQ,GACdC,EAAiB,CAAE,UAAAC,EAAW,MAAAC,CAAM,EACf,CACrB,IAAMC,EAASH,EAAG,QAAqB,UAAU,EAC3CI,EACJD,EAAO,UACPA,EAAO,cAAe,UAGxB,OAAOE,EAAc,CAACH,EAAOD,CAAS,CAAC,EACpC,KACCK,EAAI,CAAC,CAAC,CAAE,OAAAC,EAAQ,OAAAC,CAAO,EAAG,CAAE,OAAQ,CAAE,EAAAC,CAAE,CAAE,CAAC,KACzCD,EAASA,EACL,KAAK,IAAIJ,EAAQ,KAAK,IAAI,EAAGK,EAAIF,CAAM,CAAC,EACxCH,EACG,CACL,OAAAI,EACA,OAAQC,GAAKF,EAASH,CACxB,EACD,EACDM,EAAqB,CAACC,EAAGC,IACvBD,EAAE,SAAWC,EAAE,QACfD,EAAE,SAAWC,EAAE,MAChB,CACH,CACJ,CAuBO,SAASC,GACdb,EAAiBc,EACe,CADf,IAAAC,EAAAD,EAAE,SAAAE,CA5JrB,EA4JmBD,EAAcE,EAAAC,GAAdH,EAAc,CAAZ,YAEnB,IAAMI,EAAQC,EAAW,0BAA2BpB,CAAE,EAChD,CAAE,EAAAS,CAAE,EAAIY,GAAiBF,CAAK,EACpC,OAAOG,EAAM,IAAM,CACjB,IAAMC,EAAQ,IAAIC,EACZC,EAAQF,EAAM,KAAKG,GAAe,EAAGC,GAAQ,EAAI,CAAC,EAClDC,EAAQL,EACX,KACCM,GAAU,EAAGC,EAAuB,CACtC,EAGF,OAAAF,EAAM,KAAKG,GAAef,CAAO,CAAC,EAC/B,UAAU,CAGT,KAAK,CAAC,CAAE,OAAAR,CAAO,EAAG,CAAE,OAAQD,CAAO,CAAC,EAAG,CACrCY,EAAM,MAAM,OAAS,GAAGX,EAAS,EAAIC,CAAC,KACtCT,EAAG,MAAM,IAAY,GAAGO,CAAM,IAChC,EAGA,UAAW,CACTY,EAAM,MAAM,OAAS,GACrBnB,EAAG,MAAM,IAAY,EACvB,CACF,CAAC,EAGH4B,EAAM,KAAKI,GAAM,CAAC,EACf,UAAU,IAAM,CACf,QAAWC,KAAQC,EAAY,8BAA+BlC,CAAE,EAAG,CACjE,GAAI,CAACiC,EAAK,aACR,SACF,IAAME,EAAYF,EAAK,QAAqB,yBAAyB,EACrE,GAAI,OAAOE,GAAc,YAAa,CACpC,IAAM5B,EAAS0B,EAAK,UAAYE,EAAU,UACpC,CAAE,OAAA3B,CAAO,EAAI4B,GAAeD,CAAS,EAC3CA,EAAU,SAAS,CACjB,IAAK5B,EAASC,EAAS,CACzB,CAAC,CACH,CACF,CACF,CAAC,EAGH6B,GAAKH,EAA8B,kBAAmBlC,CAAE,CAAC,EACtD,KACCsC,GAASC,GAASC,EAAUD,EAAO,OAAO,EACvC,KACCE,GAAUC,EAAc,EACxBpC,EAAI,IAAMiC,CAAK,EACfI,EAAUlB,CAAK,CACjB,CACF,CACF,EACG,UAAUc,GAAS,CAClB,IAAMK,EAAQxB,EAA6B,QAAQmB,EAAM,OAAO,IAAI,EACxDnB,EAAW,qBAAqBmB,EAAM,EAAE,IAAI,EACpD,aAAa,gBAAiB,GAAGK,EAAM,OAAO,EAAE,CACtD,CAAC,EAGE7C,GAAaC,EAAIiB,CAAO,EAC5B,KACC4B,EAAIC,GAASvB,EAAM,KAAKuB,CAAK,CAAC,EAC9BC,EAAS,IAAMxB,EAAM,SAAS,CAAC,EAC/BjB,EAAIwC,GAAUE,EAAA,CAAE,IAAKhD,GAAO8C,EAAQ,CACtC,CACJ,CAAC,CACH,CCxKO,SAASG,GACdC,EAAcC,EACW,CACzB,GAAI,OAAOA,GAAS,YAAa,CAC/B,IAAMC,EAAM,gCAAgCF,CAAI,IAAIC,CAAI,GACxD,OAAOE,GAGLC,GAAqB,GAAGF,CAAG,kBAAkB,EAC1C,KACCG,GAAW,IAAMC,CAAK,EACtBC,EAAIC,IAAY,CACd,QAASA,EAAQ,QACnB,EAAE,EACFC,GAAe,CAAC,CAAC,CACnB,EAGFL,GAAkBF,CAAG,EAClB,KACCG,GAAW,IAAMC,CAAK,EACtBC,EAAIG,IAAS,CACX,MAAOA,EAAK,iBACZ,MAAOA,EAAK,WACd,EAAE,EACFD,GAAe,CAAC,CAAC,CACnB,CACJ,EACG,KACCF,EAAI,CAAC,CAACC,EAASE,CAAI,IAAOC,IAAA,GAAKH,GAAYE,EAAO,CACpD,CAGJ,KAAO,CACL,IAAMR,EAAM,gCAAgCF,CAAI,GAChD,OAAOI,GAAkBF,CAAG,EACzB,KACCK,EAAIG,IAAS,CACX,aAAcA,EAAK,YACrB,EAAE,EACFD,GAAe,CAAC,CAAC,CACnB,CACJ,CACF,CCvDO,SAASG,GACdC,EAAcC,EACW,CACzB,IAAMC,EAAM,WAAWF,CAAI,oBAAoB,mBAAmBC,CAAO,CAAC,GAC1E,OAAOE,GAA2BD,CAAG,EAClC,KACCE,GAAW,IAAMC,CAAK,EACtBC,EAAI,CAAC,CAAE,WAAAC,EAAY,YAAAC,CAAY,KAAO,CACpC,MAAOD,EACP,MAAOC,CACT,EAAE,EACFC,GAAe,CAAC,CAAC,CACnB,CACJ,CCOO,SAASC,GACdC,EACyB,CAGzB,IAAIC,EAAQD,EAAI,MAAM,qCAAqC,EAC3D,GAAIC,EAAO,CACT,GAAM,CAAC,CAAEC,EAAMC,CAAI,EAAIF,EACvB,OAAOG,GAA2BF,EAAMC,CAAI,CAC9C,CAIA,GADAF,EAAQD,EAAI,MAAM,oCAAoC,EAClDC,EAAO,CACT,GAAM,CAAC,CAAEI,EAAMC,CAAI,EAAIL,EACvB,OAAOM,GAA2BF,EAAMC,CAAI,CAC9C,CAGA,OAAOE,CACT,CCpBA,IAAIC,GAgBG,SAASC,GACdC,EACoB,CACpB,OAAOF,QAAWG,EAAM,IAAM,CAC5B,IAAMC,EAAS,SAAsB,WAAY,cAAc,EAC/D,GAAIA,EACF,OAAOC,EAAGD,CAAM,EAKhB,GADYE,GAAqB,SAAS,EAClC,OAAQ,CACd,IAAMC,EAAU,SAA0B,WAAW,EACrD,GAAI,EAAEA,GAAWA,EAAQ,QACvB,OAAOC,CACX,CAGA,OAAOC,GAAiBP,EAAG,IAAI,EAC5B,KACCQ,EAAIC,GAAS,SAAS,WAAYA,EAAO,cAAc,CAAC,CAC1D,CAEN,CAAC,EACE,KACCC,GAAW,IAAMJ,CAAK,EACtBK,EAAOF,GAAS,OAAO,KAAKA,CAAK,EAAE,OAAS,CAAC,EAC7CG,EAAIH,IAAU,CAAE,MAAAA,CAAM,EAAE,EACxBI,EAAY,CAAC,CACf,EACJ,CASO,SAASC,GACdd,EAC+B,CAC/B,IAAMe,EAAQC,EAAW,uBAAwBhB,CAAE,EACnD,OAAOC,EAAM,IAAM,CACjB,IAAMgB,EAAQ,IAAIC,EAClB,OAAAD,EAAM,UAAU,CAAC,CAAE,MAAAR,CAAM,IAAM,CAC7BM,EAAM,YAAYI,GAAkBV,CAAK,CAAC,EAC1CM,EAAM,UAAU,IAAI,+BAA+B,CACrD,CAAC,EAGMhB,GAAYC,CAAE,EAClB,KACCQ,EAAIY,GAASH,EAAM,KAAKG,CAAK,CAAC,EAC9BC,EAAS,IAAMJ,EAAM,SAAS,CAAC,EAC/BL,EAAIQ,GAAUE,EAAA,CAAE,IAAKtB,GAAOoB,EAAQ,CACtC,CACJ,CAAC,CACH,CCtDO,SAASG,GACdC,EAAiB,CAAE,UAAAC,EAAW,QAAAC,CAAQ,EACpB,CAClB,OAAOC,GAAiB,SAAS,IAAI,EAClC,KACCC,EAAU,IAAMC,GAAgBL,EAAI,CAAE,QAAAE,EAAS,UAAAD,CAAU,CAAC,CAAC,EAC3DK,EAAI,CAAC,CAAE,OAAQ,CAAE,EAAAC,CAAE,CAAE,KACZ,CACL,OAAQA,GAAK,EACf,EACD,EACDC,EAAwB,QAAQ,CAClC,CACJ,CAaO,SAASC,GACdT,EAAiBU,EACY,CAC7B,OAAOC,EAAM,IAAM,CACjB,IAAMC,EAAQ,IAAIC,EAClB,OAAAD,EAAM,UAAU,CAGd,KAAK,CAAE,OAAAE,CAAO,EAAG,CACfd,EAAG,OAASc,CACd,EAGA,UAAW,CACTd,EAAG,OAAS,EACd,CACF,CAAC,GAICe,EAAQ,wBAAwB,EAC5BC,EAAG,CAAE,OAAQ,EAAM,CAAC,EACpBjB,GAAUC,EAAIU,CAAO,GAExB,KACCO,EAAIC,GAASN,EAAM,KAAKM,CAAK,CAAC,EAC9BC,EAAS,IAAMP,EAAM,SAAS,CAAC,EAC/BN,EAAIY,GAAUE,EAAA,CAAE,IAAKpB,GAAOkB,EAAQ,CACtC,CACJ,CAAC,CACH,CCfO,SAASG,GACdC,EAAiB,CAAE,UAAAC,EAAW,QAAAC,CAAQ,EACT,CAC7B,IAAMC,EAAQ,IAAI,IAGZC,EAAUC,EAA+B,gBAAiBL,CAAE,EAClE,QAAWM,KAAUF,EAAS,CAC5B,IAAMG,EAAK,mBAAmBD,EAAO,KAAK,UAAU,CAAC,CAAC,EAChDE,EAASC,GAAmB,QAAQF,CAAE,IAAI,EAC5C,OAAOC,GAAW,aACpBL,EAAM,IAAIG,EAAQE,CAAM,CAC5B,CAGA,IAAME,EAAUR,EACb,KACCS,EAAwB,QAAQ,EAChCC,EAAI,CAAC,CAAE,OAAAC,CAAO,IAAM,CAClB,IAAMC,EAAOC,GAAoB,MAAM,EACjCC,EAAOC,EAAW,wBAAyBH,CAAI,EACrD,OAAOD,EAAS,IACdG,EAAK,UACLF,EAAK,UAET,CAAC,EACDI,GAAM,CACR,EAqFF,OAlFmBC,GAAiB,SAAS,IAAI,EAC9C,KACCR,EAAwB,QAAQ,EAGhCS,EAAUC,GAAQC,EAAM,IAAM,CAC5B,IAAIC,EAA4B,CAAC,EACjC,OAAOC,EAAG,CAAC,GAAGrB,CAAK,EAAE,OAAO,CAACsB,EAAO,CAACnB,EAAQE,CAAM,IAAM,CACvD,KAAOe,EAAK,QACGpB,EAAM,IAAIoB,EAAKA,EAAK,OAAS,CAAC,CAAC,EACnC,SAAWf,EAAO,SACzBe,EAAK,IAAI,EAOb,IAAIG,EAASlB,EAAO,UACpB,KAAO,CAACkB,GAAUlB,EAAO,eACvBA,EAASA,EAAO,cAChBkB,EAASlB,EAAO,UAIlB,IAAImB,EAASnB,EAAO,aACpB,KAAOmB,EAAQA,EAASA,EAAO,aAC7BD,GAAUC,EAAO,UAGnB,OAAOF,EAAM,IACX,CAAC,GAAGF,EAAO,CAAC,GAAGA,EAAMjB,CAAM,CAAC,EAAE,QAAQ,EACtCoB,CACF,CACF,EAAG,IAAI,GAAkC,CAAC,CAC5C,CAAC,EACE,KAGCd,EAAIa,GAAS,IAAI,IAAI,CAAC,GAAGA,CAAK,EAAE,KAAK,CAAC,CAAC,CAAEG,CAAC,EAAG,CAAC,CAAEC,CAAC,IAAMD,EAAIC,CAAC,CAAC,CAAC,EAC9DC,GAAkBpB,CAAO,EAGzBU,EAAU,CAAC,CAACK,EAAOM,CAAM,IAAM9B,EAC5B,KACC+B,GAAK,CAAC,CAACC,EAAMC,CAAI,EAAG,CAAE,OAAQ,CAAE,EAAAC,CAAE,EAAG,KAAAC,CAAK,IAAM,CAC9C,IAAMC,EAAOF,EAAIC,EAAK,QAAU,KAAK,MAAMf,EAAK,MAAM,EAGtD,KAAOa,EAAK,QAAQ,CAClB,GAAM,CAAC,CAAER,CAAM,EAAIQ,EAAK,CAAC,EACzB,GAAIR,EAASK,EAASI,GAAKE,EACzBJ,EAAO,CAAC,GAAGA,EAAMC,EAAK,MAAM,CAAE,MAE9B,MAEJ,CAGA,KAAOD,EAAK,QAAQ,CAClB,GAAM,CAAC,CAAEP,CAAM,EAAIO,EAAKA,EAAK,OAAS,CAAC,EACvC,GAAIP,EAASK,GAAUI,GAAK,CAACE,EAC3BH,EAAO,CAACD,EAAK,IAAI,EAAI,GAAGC,CAAI,MAE5B,MAEJ,CAGA,MAAO,CAACD,EAAMC,CAAI,CACpB,EAAG,CAAC,CAAC,EAAG,CAAC,GAAGT,CAAK,CAAC,CAAC,EACnBa,EAAqB,CAACV,EAAGC,IACvBD,EAAE,CAAC,IAAMC,EAAE,CAAC,GACZD,EAAE,CAAC,IAAMC,EAAE,CAAC,CACb,CACH,CACF,CACF,CACF,CACF,EAIC,KACCjB,EAAI,CAAC,CAACqB,EAAMC,CAAI,KAAO,CACrB,KAAMD,EAAK,IAAI,CAAC,CAACV,CAAI,IAAMA,CAAI,EAC/B,KAAMW,EAAK,IAAI,CAAC,CAACX,CAAI,IAAMA,CAAI,CACjC,EAAE,EAGFgB,EAAU,CAAE,KAAM,CAAC,EAAG,KAAM,CAAC,CAAE,CAAC,EAChCC,GAAY,EAAG,CAAC,EAChB5B,EAAI,CAAC,CAAC,EAAGiB,CAAC,IAGJ,EAAE,KAAK,OAASA,EAAE,KAAK,OAClB,CACL,KAAMA,EAAE,KAAK,MAAM,KAAK,IAAI,EAAG,EAAE,KAAK,OAAS,CAAC,EAAGA,EAAE,KAAK,MAAM,EAChE,KAAM,CAAC,CACT,EAIO,CACL,KAAMA,EAAE,KAAK,MAAM,EAAE,EACrB,KAAMA,EAAE,KAAK,MAAM,EAAGA,EAAE,KAAK,OAAS,EAAE,KAAK,MAAM,CACrD,CAEH,CACH,CACJ,CAYO,SAASY,GACdzC,EAAiB,CAAE,UAAAC,EAAW,QAAAC,EAAS,MAAAwC,EAAO,QAAAC,CAAQ,EACd,CACxC,OAAOrB,EAAM,IAAM,CACjB,IAAMsB,EAAQ,IAAIC,EACZC,EAAQF,EAAM,KAAKG,GAAe,EAAGC,GAAQ,EAAI,CAAC,EAoBxD,GAnBAJ,EAAM,UAAU,CAAC,CAAE,KAAAX,EAAM,KAAAC,CAAK,IAAM,CAGlC,OAAW,CAAC5B,CAAM,IAAK4B,EACrB5B,EAAO,UAAU,OAAO,sBAAsB,EAC9CA,EAAO,UAAU,OAAO,sBAAsB,EAIhD,OAAW,CAACmB,EAAO,CAACnB,CAAM,CAAC,IAAK2B,EAAK,QAAQ,EAC3C3B,EAAO,UAAU,IAAI,sBAAsB,EAC3CA,EAAO,UAAU,OACf,uBACAmB,IAAUQ,EAAK,OAAS,CAC1B,CAEJ,CAAC,EAGGgB,EAAQ,YAAY,EAAG,CAGzB,IAAMC,EAAUC,EACdlD,EAAU,KAAKmD,GAAa,CAAC,EAAGxC,EAAI,IAAG,EAAY,CAAC,EACpDX,EAAU,KAAKmD,GAAa,GAAG,EAAGxC,EAAI,IAAM,QAAiB,CAAC,CAChE,EAGAgC,EACG,KACCS,EAAO,CAAC,CAAE,KAAApB,CAAK,IAAMA,EAAK,OAAS,CAAC,EACpCH,GAAkBY,EAAM,KAAKY,GAAUC,EAAc,CAAC,CAAC,EACvDC,GAAeN,CAAO,CACxB,EACG,UAAU,CAAC,CAAC,CAAC,CAAE,KAAAjB,CAAK,CAAC,EAAGwB,CAAQ,IAAM,CACrC,GAAM,CAACnD,CAAM,EAAI2B,EAAKA,EAAK,OAAS,CAAC,EACrC,GAAI3B,EAAO,aAAc,CAGvB,IAAMoD,EAAYC,GAAoBrD,CAAM,EAC5C,GAAI,OAAOoD,GAAc,YAAa,CACpC,IAAMhC,EAASpB,EAAO,UAAYoD,EAAU,UACtC,CAAE,OAAA7C,CAAO,EAAI+C,GAAeF,CAAS,EAC3CA,EAAU,SAAS,CACjB,IAAKhC,EAASb,EAAS,EACvB,SAAA4C,CACF,CAAC,CACH,CACF,CACF,CAAC,CACP,CAGA,OAAIR,EAAQ,qBAAqB,GAC/BhD,EACG,KACC4D,EAAUf,CAAK,EACfnC,EAAwB,QAAQ,EAChCyC,GAAa,GAAG,EAChBU,GAAK,CAAC,EACND,EAAUlB,EAAQ,KAAKmB,GAAK,CAAC,CAAC,CAAC,EAC/BC,GAAO,CAAE,MAAO,GAAI,CAAC,EACrBP,GAAeZ,CAAK,CACtB,EACG,UAAU,CAAC,CAAC,CAAE,CAAE,KAAAX,CAAK,CAAC,IAAM,CAC3B,IAAM+B,EAAMC,GAAY,EAGlB3D,EAAS2B,EAAKA,EAAK,OAAS,CAAC,EACnC,GAAI3B,GAAUA,EAAO,OAAQ,CAC3B,GAAM,CAAC4D,CAAM,EAAI5D,EACX,CAAE,KAAA6D,CAAK,EAAI,IAAI,IAAID,EAAO,IAAI,EAChCF,EAAI,OAASG,IACfH,EAAI,KAAOG,EACX,QAAQ,aAAa,CAAC,EAAG,GAAI,GAAGH,CAAG,EAAE,EAIzC,MACEA,EAAI,KAAO,GACX,QAAQ,aAAa,CAAC,EAAG,GAAI,GAAGA,CAAG,EAAE,CAEzC,CAAC,EAGAjE,GAAqBC,EAAI,CAAE,UAAAC,EAAW,QAAAC,CAAQ,CAAC,EACnD,KACCkE,EAAIC,GAASzB,EAAM,KAAKyB,CAAK,CAAC,EAC9BC,EAAS,IAAM1B,EAAM,SAAS,CAAC,EAC/BhC,EAAIyD,GAAUE,EAAA,CAAE,IAAKvE,GAAOqE,EAAQ,CACtC,CACJ,CAAC,CACH,CC9RO,SAASG,GACdC,EAAkB,CAAE,UAAAC,EAAW,MAAAC,EAAO,QAAAC,CAAQ,EACvB,CAGvB,IAAMC,EAAaH,EAChB,KACCI,EAAI,CAAC,CAAE,OAAQ,CAAE,EAAAC,CAAE,CAAE,IAAMA,CAAC,EAC5BC,GAAY,EAAG,CAAC,EAChBF,EAAI,CAAC,CAACG,EAAGC,CAAC,IAAMD,EAAIC,GAAKA,EAAI,CAAC,EAC9BC,EAAqB,CACvB,EAGIC,EAAUT,EACb,KACCG,EAAI,CAAC,CAAE,OAAAO,CAAO,IAAMA,CAAM,CAC5B,EAGF,OAAOC,EAAc,CAACF,EAASP,CAAU,CAAC,EACvC,KACCC,EAAI,CAAC,CAACO,EAAQE,CAAS,IAAM,EAAEF,GAAUE,EAAU,EACnDJ,EAAqB,EACrBK,EAAUZ,EAAQ,KAAKa,GAAK,CAAC,CAAC,CAAC,EAC/BC,GAAQ,EAAI,EACZC,GAAO,CAAE,MAAO,GAAI,CAAC,EACrBb,EAAIc,IAAW,CAAE,OAAAA,CAAO,EAAE,CAC5B,CACJ,CAYO,SAASC,GACdC,EAAiB,CAAE,UAAApB,EAAW,QAAAqB,EAAS,MAAApB,EAAO,QAAAC,CAAQ,EACpB,CAClC,IAAMoB,EAAQ,IAAIC,EACZC,EAAQF,EAAM,KAAKG,GAAe,EAAGT,GAAQ,EAAI,CAAC,EACxD,OAAAM,EAAM,UAAU,CAGd,KAAK,CAAE,OAAAJ,CAAO,EAAG,CACfE,EAAG,OAASF,EACRA,GACFE,EAAG,aAAa,WAAY,IAAI,EAChCA,EAAG,KAAK,GAERA,EAAG,gBAAgB,UAAU,CAEjC,EAGA,UAAW,CACTA,EAAG,MAAM,IAAM,GACfA,EAAG,OAAS,GACZA,EAAG,gBAAgB,UAAU,CAC/B,CACF,CAAC,EAGDC,EACG,KACCP,EAAUU,CAAK,EACfE,EAAwB,QAAQ,CAClC,EACG,UAAU,CAAC,CAAE,OAAAC,CAAO,IAAM,CACzBP,EAAG,MAAM,IAAM,GAAGO,EAAS,EAAE,IAC/B,CAAC,EAGLC,EAAUR,EAAI,OAAO,EAClB,UAAUS,GAAM,CACfA,EAAG,eAAe,EAClB,OAAO,SAAS,CAAE,IAAK,CAAE,CAAC,CAC5B,CAAC,EAGI/B,GAAesB,EAAI,CAAE,UAAApB,EAAW,MAAAC,EAAO,QAAAC,CAAQ,CAAC,EACpD,KACC4B,EAAIC,GAAST,EAAM,KAAKS,CAAK,CAAC,EAC9BC,EAAS,IAAMV,EAAM,SAAS,CAAC,EAC/BlB,EAAI2B,GAAUE,EAAA,CAAE,IAAKb,GAAOW,EAAQ,CACtC,CACJ,CCtHO,SAASG,GACd,CAAE,UAAAC,CAAU,EACN,CACNA,EACG,KACCC,EAAU,IAAMC,EAAY,cAAc,CAAC,EAC3CC,GAASC,GAAMC,GAAuBD,CAAE,EACrC,KACCE,EAAUN,EAAU,KAAKO,GAAK,CAAC,CAAC,CAAC,EACjCC,EAAOC,GAAWA,CAAO,EACzBC,EAAI,IAAMN,CAAE,EACZO,GAAK,CAAC,CACR,CACF,EACAH,EAAOJ,GAAMA,EAAG,YAAcA,EAAG,WAAW,EAC5CD,GAASC,GAAM,CACb,IAAMQ,EAAOR,EAAG,UACVS,EAAOT,EAAG,QAAQ,GAAG,GAAKA,EAChC,OAAAS,EAAK,MAAQD,EAGNE,GAAaD,CAAI,EACrB,KACCP,EAAUN,EAAU,KAAKO,GAAK,CAAC,CAAC,CAAC,EACjCQ,EAAS,IAAMF,EAAK,gBAAgB,OAAO,CAAC,CAC9C,CACJ,CAAC,CACH,EACG,UAAU,EAGfb,EACG,KACCC,EAAU,IAAMC,EAAY,YAAY,CAAC,EACzCC,GAASC,GAAMU,GAAaV,CAAE,CAAC,CACjC,EACG,UAAU,CACjB,CC3CO,SAASY,GACd,CAAE,UAAAC,EAAW,QAAAC,CAAQ,EACf,CACND,EACG,KACCE,EAAU,IAAMC,EACd,2BACF,CAAC,EACDC,EAAIC,GAAM,CACRA,EAAG,cAAgB,GACnBA,EAAG,QAAU,EACf,CAAC,EACDC,GAASD,GAAME,EAAUF,EAAI,QAAQ,EAClC,KACCG,GAAU,IAAMH,EAAG,UAAU,SAAS,0BAA0B,CAAC,EACjEI,EAAI,IAAMJ,CAAE,CACd,CACF,EACAK,GAAeT,CAAO,CACxB,EACG,UAAU,CAAC,CAACI,EAAIM,CAAM,IAAM,CAC3BN,EAAG,UAAU,OAAO,0BAA0B,EAC1CM,IACFN,EAAG,QAAU,GACjB,CAAC,CACP,CC9BA,SAASO,IAAyB,CAChC,MAAO,qBAAqB,KAAK,UAAU,SAAS,CACtD,CAiBO,SAASC,GACd,CAAE,UAAAC,CAAU,EACN,CACNA,EACG,KACCC,EAAU,IAAMC,EAAY,qBAAqB,CAAC,EAClDC,EAAIC,GAAMA,EAAG,gBAAgB,mBAAmB,CAAC,EACjDC,EAAOP,EAAa,EACpBQ,GAASF,GAAMG,EAAUH,EAAI,YAAY,EACtC,KACCI,EAAI,IAAMJ,CAAE,CACd,CACF,CACF,EACG,UAAUA,GAAM,CACf,IAAMK,EAAML,EAAG,UAGXK,IAAQ,EACVL,EAAG,UAAY,EAGNK,EAAML,EAAG,eAAiBA,EAAG,eACtCA,EAAG,UAAYK,EAAM,EAEzB,CAAC,CACP,CCpCO,SAASC,GACd,CAAE,UAAAC,EAAW,QAAAC,CAAQ,EACf,CACNC,EAAc,CAACC,GAAY,QAAQ,EAAGF,CAAO,CAAC,EAC3C,KACCG,EAAI,CAAC,CAACC,EAAQC,CAAM,IAAMD,GAAU,CAACC,CAAM,EAC3CC,EAAUF,GAAUG,EAAGH,CAAM,EAC1B,KACCI,GAAMJ,EAAS,IAAM,GAAG,CAC1B,CACF,EACAK,GAAeV,CAAS,CAC1B,EACG,UAAU,CAAC,CAACK,EAAQ,CAAE,OAAQ,CAAE,EAAAM,CAAE,CAAC,CAAC,IAAM,CACzC,GAAIN,EACF,SAAS,KAAK,aAAa,qBAAsB,EAAE,EACnD,SAAS,KAAK,MAAM,IAAM,IAAIM,CAAC,SAC1B,CACL,IAAMC,EAAQ,GAAK,SAAS,SAAS,KAAK,MAAM,IAAK,EAAE,EACvD,SAAS,KAAK,gBAAgB,oBAAoB,EAClD,SAAS,KAAK,MAAM,IAAM,GACtBA,GACF,OAAO,SAAS,EAAGA,CAAK,CAC5B,CACF,CAAC,CACP,CC7DK,OAAO,UACV,OAAO,QAAU,SAAUC,EAAa,CACtC,IAAMC,EAA2B,CAAC,EAClC,QAAWC,KAAO,OAAO,KAAKF,CAAG,EAE/BC,EAAK,KAAK,CAACC,EAAKF,EAAIE,CAAG,CAAC,CAAC,EAG3B,OAAOD,CACT,GAGG,OAAO,SACV,OAAO,OAAS,SAAUD,EAAa,CACrC,IAAMC,EAAiB,CAAC,EACxB,QAAWC,KAAO,OAAO,KAAKF,CAAG,EAE/BC,EAAK,KAAKD,EAAIE,CAAG,CAAC,EAGpB,OAAOD,CACT,GAKE,OAAO,SAAY,cAGhB,QAAQ,UAAU,WACrB,QAAQ,UAAU,SAAW,SAC3BE,EAA8BC,EACxB,CACF,OAAOD,GAAM,UACf,KAAK,WAAaA,EAAE,KACpB,KAAK,UAAYA,EAAE,MAEnB,KAAK,WAAaA,EAClB,KAAK,UAAYC,EAErB,GAGG,QAAQ,UAAU,cACrB,QAAQ,UAAU,YAAc,YAC3BC,EACG,CACN,IAAMC,EAAS,KAAK,WACpB,GAAIA,EAAQ,CACND,EAAM,SAAW,GACnBC,EAAO,YAAY,IAAI,EAGzB,QAASC,EAAIF,EAAM,OAAS,EAAGE,GAAK,EAAGA,IAAK,CAC1C,IAAIC,EAAOH,EAAME,CAAC,EACd,OAAOC,GAAS,SAClBA,EAAO,SAAS,eAAeA,CAAI,EAC5BA,EAAK,YACZA,EAAK,WAAW,YAAYA,CAAI,EAG7BD,EAGHD,EAAO,aAAa,KAAK,gBAAkBE,CAAI,EAF/CF,EAAO,aAAaE,EAAM,IAAI,CAGlC,CACF,CACF,IpMMJ,SAASC,IAA4C,CACnD,OAAI,SAAS,WAAa,QACjBC,GACL,GAAG,IAAI,IAAI,yBAA0BC,GAAO,IAAI,CAAC,EACnD,EACG,KAECC,EAAI,IAAM,OAAO,EACjBC,EAAY,CAAC,CACf,EAEKC,GACL,IAAI,IAAI,2BAA4BH,GAAO,IAAI,CACjD,CAEJ,CAOA,SAAS,gBAAgB,UAAU,OAAO,OAAO,EACjD,SAAS,gBAAgB,UAAU,IAAI,IAAI,EAG3C,IAAMI,GAAYC,GAAc,EAC1BC,GAAYC,GAAc,EAC1BC,GAAYC,GAAoBH,EAAS,EACzCI,GAAYC,GAAc,EAG1BC,GAAYC,GAAc,EAC1BC,GAAYC,GAAW,oBAAoB,EAC3CC,GAAYD,GAAW,qBAAqB,EAC5CE,GAAYC,GAAW,EAGvBlB,GAASmB,GAAc,EACvBC,GAAS,SAAS,MAAM,UAAU,QAAQ,EAC5CtB,GAAiB,EACjBuB,GAGEC,GAAS,IAAIC,EACnBC,GAAiB,CAAE,OAAAF,EAAO,CAAC,EAG3B,IAAMG,GAAY,IAAIF,EAGlBG,EAAQ,oBAAoB,GAC9BC,GAAuB,CAAE,UAAArB,GAAW,UAAAM,GAAW,UAAAa,EAAU,CAAC,EACvD,UAAUrB,EAAS,EAzJxB,IAAAwB,KA4JIA,GAAA5B,GAAO,UAAP,YAAA4B,GAAgB,YAAa,QAC/BC,GAAqB,CAAE,UAAAzB,EAAU,CAAC,EAGpC0B,EAAMxB,GAAWE,EAAO,EACrB,KACCuB,GAAM,GAAG,CACX,EACG,UAAU,IAAM,CACfC,GAAU,SAAU,EAAK,EACzBA,GAAU,SAAU,EAAK,CAC3B,CAAC,EAGLtB,GACG,KACCuB,EAAO,CAAC,CAAE,KAAAC,CAAK,IAAMA,IAAS,QAAQ,CACxC,EACG,UAAUC,GAAO,CAChB,OAAQA,EAAI,KAAM,CAGhB,IAAK,IACL,IAAK,IACH,IAAMC,EAAOC,GAAoC,gBAAgB,EAC7D,OAAOD,GAAS,aAClBE,GAAYF,CAAI,EAClB,MAGF,IAAK,IACL,IAAK,IACH,IAAMG,EAAOF,GAAoC,gBAAgB,EAC7D,OAAOE,GAAS,aAClBD,GAAYC,CAAI,EAClB,MAGF,IAAK,QACH,IAAMC,EAASC,GAAiB,EAC5BD,aAAkB,kBACpBA,EAAO,MAAM,CACnB,CACF,CAAC,EAGLE,GAAc,CAAE,UAAAtC,EAAU,CAAC,EAC3BuC,GAAmB,CAAE,UAAAvC,GAAW,QAAAU,EAAQ,CAAC,EACzC8B,GAAe,CAAE,UAAAxC,EAAU,CAAC,EAC5ByC,GAAgB,CAAE,UAAAjC,GAAW,QAAAE,EAAQ,CAAC,EAGtC,IAAMgC,GAAUC,GAAYC,GAAoB,QAAQ,EAAG,CAAE,UAAApC,EAAU,CAAC,EAClEqC,GAAQ7C,GACX,KACCH,EAAI,IAAM+C,GAAoB,MAAM,CAAC,EACrCE,EAAUC,GAAMC,GAAUD,EAAI,CAAE,UAAAvC,GAAW,QAAAkC,EAAQ,CAAC,CAAC,EACrD5C,EAAY,CAAC,CACf,EAGImD,GAAWvB,EAGf,GAAGwB,GAAqB,SAAS,EAC9B,IAAIH,GAAMI,GAAaJ,EAAI,CAAE,QAAA3C,EAAQ,CAAC,CAAC,EAG1C,GAAG8C,GAAqB,QAAQ,EAC7B,IAAIH,GAAMK,GAAYL,EAAI,CAAE,OAAA7B,EAAO,CAAC,CAAC,EAGxC,GAAGgC,GAAqB,QAAQ,EAC7B,IAAIH,GAAMM,GAAYN,EAAI,CAAE,UAAAvC,GAAW,QAAAkC,GAAS,MAAAG,EAAM,CAAC,CAAC,EAG3D,GAAGK,GAAqB,SAAS,EAC9B,IAAIH,GAAMO,GAAaP,CAAE,CAAC,EAG7B,GAAGG,GAAqB,UAAU,EAC/B,IAAIH,GAAMQ,GAAcR,EAAI,CAAE,UAAA1B,EAAU,CAAC,CAAC,EAG7C,GAAG6B,GAAqB,QAAQ,EAC7B,IAAIH,GAAMS,GAAYT,EAAI,CAAE,OAAA/B,GAAQ,UAAAV,EAAU,CAAC,CAAC,EAGnD,GAAG4C,GAAqB,QAAQ,EAC7B,IAAIH,GAAMU,GAAYV,CAAE,CAAC,CAC9B,EAGMW,GAAWC,EAAM,IAAMjC,EAG3B,GAAGwB,GAAqB,UAAU,EAC/B,IAAIH,GAAMa,GAAcb,CAAE,CAAC,EAG9B,GAAGG,GAAqB,SAAS,EAC9B,IAAIH,GAAMc,GAAad,EAAI,CAAE,UAAAvC,GAAW,QAAAJ,GAAS,OAAAS,EAAO,CAAC,CAAC,EAG7D,GAAGqC,GAAqB,SAAS,EAC9B,IAAIH,GAAMzB,EAAQ,kBAAkB,EACjCwC,GAAoBf,EAAI,CAAE,OAAA/B,GAAQ,UAAAd,EAAU,CAAC,EAC7C6D,CACJ,EAGF,GAAGb,GAAqB,cAAc,EACnC,IAAIH,GAAMiB,GAAiBjB,EAAI,CAAE,UAAAvC,GAAW,QAAAkC,EAAQ,CAAC,CAAC,EAGzD,GAAGQ,GAAqB,SAAS,EAC9B,IAAIH,GAAMA,EAAG,aAAa,cAAc,IAAM,aAC3CkB,GAAGrD,GAAS,IAAMsD,GAAanB,EAAI,CAAE,UAAAvC,GAAW,QAAAkC,GAAS,MAAAG,EAAM,CAAC,CAAC,EACjEoB,GAAGvD,GAAS,IAAMwD,GAAanB,EAAI,CAAE,UAAAvC,GAAW,QAAAkC,GAAS,MAAAG,EAAM,CAAC,CAAC,CACrE,EAGF,GAAGK,GAAqB,MAAM,EAC3B,IAAIH,GAAMoB,GAAUpB,EAAI,CAAE,UAAAvC,GAAW,QAAAkC,EAAQ,CAAC,CAAC,EAGlD,GAAGQ,GAAqB,KAAK,EAC1B,IAAIH,GAAMqB,GAAqBrB,EAAI,CAClC,UAAAvC,GAAW,QAAAkC,GAAS,MAAAG,GAAO,QAAAzC,EAC7B,CAAC,CAAC,EAGJ,GAAG8C,GAAqB,KAAK,EAC1B,IAAIH,GAAMsB,GAAetB,EAAI,CAAE,UAAAvC,GAAW,QAAAkC,GAAS,MAAAG,GAAO,QAAAzC,EAAQ,CAAC,CAAC,CACzE,CAAC,EAGKkE,GAAatE,GAChB,KACC8C,EAAU,IAAMY,EAAQ,EACxBa,GAAUtB,EAAQ,EAClBnD,EAAY,CAAC,CACf,EAGFwE,GAAW,UAAU,EAMrB,OAAO,UAAatE,GACpB,OAAO,UAAaE,GACpB,OAAO,QAAaE,GACpB,OAAO,UAAaE,GACpB,OAAO,UAAaE,GACpB,OAAO,QAAaE,GACpB,OAAO,QAAaE,GACpB,OAAO,OAAaC,GACpB,OAAO,OAAaK,GACpB,OAAO,UAAaG,GACpB,OAAO,WAAaiD",
+  "names": ["require_focus_visible", "__commonJSMin", "exports", "module", "global", "factory", "applyFocusVisiblePolyfill", "scope", "hadKeyboardEvent", "hadFocusVisibleRecently", "hadFocusVisibleRecentlyTimeout", "inputTypesAllowlist", "isValidFocusTarget", "el", "focusTriggersKeyboardModality", "type", "tagName", "addFocusVisibleClass", "removeFocusVisibleClass", "onKeyDown", "e", "onPointerDown", "onFocus", "onBlur", "onVisibilityChange", "addInitialPointerMoveListeners", "onInitialPointerMove", "removeInitialPointerMoveListeners", "event", "error", "require_clipboard", "__commonJSMin", "exports", "module", "root", "factory", "__webpack_modules__", "__unused_webpack_module", "__webpack_exports__", "__webpack_require__", "clipboard", "tiny_emitter", "tiny_emitter_default", "listen", "listen_default", "src_select", "select_default", "command", "type", "err", "ClipboardActionCut", "target", "selectedText", "actions_cut", "createFakeElement", "value", "isRTL", "fakeElement", "yPosition", "fakeCopyAction", "options", "ClipboardActionCopy", "actions_copy", "_typeof", "obj", "ClipboardActionDefault", "_options$action", "action", "container", "text", "actions_default", "clipboard_typeof", "_classCallCheck", "instance", "Constructor", "_defineProperties", "props", "i", "descriptor", "_createClass", "protoProps", "staticProps", "_inherits", "subClass", "superClass", "_setPrototypeOf", "o", "p", "_createSuper", "Derived", "hasNativeReflectConstruct", "_isNativeReflectConstruct", "Super", "_getPrototypeOf", "result", "NewTarget", "_possibleConstructorReturn", "self", "call", "_assertThisInitialized", "e", "getAttributeValue", "suffix", "element", "attribute", "Clipboard", "_Emitter", "_super", "trigger", "_this", "_this2", "selector", "actions", "support", "DOCUMENT_NODE_TYPE", "proto", "closest", "__unused_webpack_exports", "_delegate", "callback", "useCapture", "listenerFn", "listener", "delegate", "elements", "is", "listenNode", "listenNodeList", "listenSelector", "node", "nodeList", "select", "isReadOnly", "selection", "range", "E", "name", "ctx", "data", "evtArr", "len", "evts", "liveEvents", "__webpack_module_cache__", "moduleId", "getter", "definition", "key", "prop", "require_escape_html", "__commonJSMin", "exports", "module", "matchHtmlRegExp", "escapeHtml", "string", "str", "match", "escape", "html", "index", "lastIndex", "import_focus_visible", "extendStatics", "d", "b", "p", "__extends", "__", "__awaiter", "thisArg", "_arguments", "P", "generator", "adopt", "value", "resolve", "reject", "fulfilled", "step", "e", "rejected", "result", "__generator", "body", "_", "t", "f", "y", "g", "verb", "n", "v", "op", "__values", "o", "s", "m", "i", "__read", "n", "r", "ar", "e", "error", "__spreadArray", "to", "from", "pack", "i", "l", "ar", "__await", "v", "__asyncGenerator", "thisArg", "_arguments", "generator", "g", "q", "verb", "n", "a", "b", "resume", "step", "e", "settle", "r", "fulfill", "reject", "value", "f", "__asyncValues", "o", "m", "i", "__values", "verb", "n", "v", "resolve", "reject", "settle", "d", "isFunction", "value", "createErrorClass", "createImpl", "_super", "instance", "ctorFunc", "UnsubscriptionError", "createErrorClass", "_super", "errors", "err", "i", "arrRemove", "arr", "item", "index", "Subscription", "initialTeardown", "errors", "_parentage", "_parentage_1", "__values", "_parentage_1_1", "parent_1", "initialFinalizer", "isFunction", "e", "UnsubscriptionError", "_finalizers", "_finalizers_1", "_finalizers_1_1", "finalizer", "execFinalizer", "err", "__spreadArray", "__read", "teardown", "_a", "parent", "arrRemove", "empty", "EMPTY_SUBSCRIPTION", "Subscription", "isSubscription", "value", "isFunction", "execFinalizer", "finalizer", "config", "timeoutProvider", "handler", "timeout", "args", "_i", "delegate", "__spreadArray", "__read", "handle", "reportUnhandledError", "err", "timeoutProvider", "onUnhandledError", "config", "noop", "COMPLETE_NOTIFICATION", "createNotification", "errorNotification", "error", "nextNotification", "value", "kind", "context", "errorContext", "cb", "config", "isRoot", "_a", "errorThrown", "error", "captureError", "err", "Subscriber", "_super", "__extends", "destination", "_this", "isSubscription", "EMPTY_OBSERVER", "next", "error", "complete", "SafeSubscriber", "value", "handleStoppedNotification", "nextNotification", "err", "errorNotification", "COMPLETE_NOTIFICATION", "Subscription", "_bind", "bind", "fn", "thisArg", "ConsumerObserver", "partialObserver", "value", "error", "handleUnhandledError", "err", "SafeSubscriber", "_super", "__extends", "observerOrNext", "complete", "_this", "isFunction", "context_1", "config", "Subscriber", "handleUnhandledError", "error", "config", "captureError", "reportUnhandledError", "defaultErrorHandler", "err", "handleStoppedNotification", "notification", "subscriber", "onStoppedNotification", "timeoutProvider", "EMPTY_OBSERVER", "noop", "observable", "identity", "x", "pipe", "fns", "_i", "pipeFromArray", "identity", "input", "prev", "fn", "Observable", "subscribe", "operator", "observable", "observerOrNext", "error", "complete", "_this", "subscriber", "isSubscriber", "SafeSubscriber", "errorContext", "_a", "source", "sink", "err", "next", "promiseCtor", "getPromiseCtor", "resolve", "reject", "value", "operations", "_i", "pipeFromArray", "x", "getPromiseCtor", "promiseCtor", "_a", "config", "isObserver", "value", "isFunction", "isSubscriber", "Subscriber", "isSubscription", "hasLift", "source", "isFunction", "operate", "init", "liftedSource", "err", "createOperatorSubscriber", "destination", "onNext", "onComplete", "onError", "onFinalize", "OperatorSubscriber", "_super", "__extends", "shouldUnsubscribe", "_this", "value", "err", "closed_1", "_a", "Subscriber", "animationFrameProvider", "callback", "request", "cancel", "delegate", "handle", "timestamp", "Subscription", "args", "_i", "__spreadArray", "__read", "ObjectUnsubscribedError", "createErrorClass", "_super", "Subject", "_super", "__extends", "_this", "operator", "subject", "AnonymousSubject", "ObjectUnsubscribedError", "value", "errorContext", "_b", "__values", "_c", "observer", "err", "observers", "_a", "subscriber", "hasError", "isStopped", "EMPTY_SUBSCRIPTION", "Subscription", "arrRemove", "thrownError", "observable", "Observable", "destination", "source", "AnonymousSubject", "_super", "__extends", "destination", "source", "_this", "value", "_b", "_a", "err", "subscriber", "EMPTY_SUBSCRIPTION", "Subject", "dateTimestampProvider", "ReplaySubject", "_super", "__extends", "_bufferSize", "_windowTime", "_timestampProvider", "dateTimestampProvider", "_this", "value", "_a", "isStopped", "_buffer", "_infiniteTimeWindow", "subscriber", "subscription", "copy", "i", "adjustedBufferSize", "now", "last", "Subject", "Action", "_super", "__extends", "scheduler", "work", "state", "delay", "Subscription", "intervalProvider", "handler", "timeout", "args", "_i", "delegate", "__spreadArray", "__read", "handle", "AsyncAction", "_super", "__extends", "scheduler", "work", "_this", "state", "delay", "id", "_a", "_id", "intervalProvider", "_scheduler", "error", "_delay", "errored", "errorValue", "e", "actions", "arrRemove", "Action", "Scheduler", "schedulerActionCtor", "now", "work", "delay", "state", "dateTimestampProvider", "AsyncScheduler", "_super", "__extends", "SchedulerAction", "now", "Scheduler", "_this", "action", "actions", "error", "asyncScheduler", "AsyncScheduler", "AsyncAction", "async", "AnimationFrameAction", "_super", "__extends", "scheduler", "work", "_this", "id", "delay", "animationFrameProvider", "actions", "_a", "AsyncAction", "AnimationFrameScheduler", "_super", "__extends", "action", "flushId", "actions", "error", "AsyncScheduler", "animationFrameScheduler", "AnimationFrameScheduler", "AnimationFrameAction", "EMPTY", "Observable", "subscriber", "isScheduler", "value", "isFunction", "last", "arr", "popResultSelector", "args", "isFunction", "popScheduler", "isScheduler", "popNumber", "defaultValue", "isArrayLike", "x", "isPromise", "value", "isFunction", "isInteropObservable", "input", "isFunction", "observable", "isAsyncIterable", "obj", "isFunction", "createInvalidObservableTypeError", "input", "getSymbolIterator", "iterator", "isIterable", "input", "isFunction", "iterator", "readableStreamLikeToAsyncGenerator", "readableStream", "reader", "__await", "_a", "_b", "value", "done", "isReadableStreamLike", "obj", "isFunction", "innerFrom", "input", "Observable", "isInteropObservable", "fromInteropObservable", "isArrayLike", "fromArrayLike", "isPromise", "fromPromise", "isAsyncIterable", "fromAsyncIterable", "isIterable", "fromIterable", "isReadableStreamLike", "fromReadableStreamLike", "createInvalidObservableTypeError", "obj", "subscriber", "obs", "observable", "isFunction", "array", "i", "promise", "value", "err", "reportUnhandledError", "iterable", "iterable_1", "__values", "iterable_1_1", "asyncIterable", "process", "readableStream", "readableStreamLikeToAsyncGenerator", "asyncIterable_1", "__asyncValues", "asyncIterable_1_1", "executeSchedule", "parentSubscription", "scheduler", "work", "delay", "repeat", "scheduleSubscription", "observeOn", "scheduler", "delay", "operate", "source", "subscriber", "createOperatorSubscriber", "value", "executeSchedule", "err", "subscribeOn", "scheduler", "delay", "operate", "source", "subscriber", "scheduleObservable", "input", "scheduler", "innerFrom", "subscribeOn", "observeOn", "schedulePromise", "input", "scheduler", "innerFrom", "subscribeOn", "observeOn", "scheduleArray", "input", "scheduler", "Observable", "subscriber", "i", "scheduleIterable", "input", "scheduler", "Observable", "subscriber", "iterator", "executeSchedule", "value", "done", "_a", "err", "isFunction", "scheduleAsyncIterable", "input", "scheduler", "Observable", "subscriber", "executeSchedule", "iterator", "result", "scheduleReadableStreamLike", "input", "scheduler", "scheduleAsyncIterable", "readableStreamLikeToAsyncGenerator", "scheduled", "input", "scheduler", "isInteropObservable", "scheduleObservable", "isArrayLike", "scheduleArray", "isPromise", "schedulePromise", "isAsyncIterable", "scheduleAsyncIterable", "isIterable", "scheduleIterable", "isReadableStreamLike", "scheduleReadableStreamLike", "createInvalidObservableTypeError", "from", "input", "scheduler", "scheduled", "innerFrom", "of", "args", "_i", "scheduler", "popScheduler", "from", "throwError", "errorOrErrorFactory", "scheduler", "errorFactory", "isFunction", "init", "subscriber", "Observable", "EmptyError", "createErrorClass", "_super", "isValidDate", "value", "map", "project", "thisArg", "operate", "source", "subscriber", "index", "createOperatorSubscriber", "value", "isArray", "callOrApply", "fn", "args", "__spreadArray", "__read", "mapOneOrManyArgs", "map", "isArray", "getPrototypeOf", "objectProto", "getKeys", "argsArgArrayOrObject", "args", "first_1", "isPOJO", "keys", "key", "obj", "createObject", "keys", "values", "result", "key", "i", "combineLatest", "args", "_i", "scheduler", "popScheduler", "resultSelector", "popResultSelector", "_a", "argsArgArrayOrObject", "observables", "keys", "from", "result", "Observable", "combineLatestInit", "values", "createObject", "identity", "mapOneOrManyArgs", "valueTransform", "subscriber", "maybeSchedule", "length", "active", "remainingFirstValues", "i", "source", "hasFirstValue", "createOperatorSubscriber", "value", "execute", "subscription", "executeSchedule", "mergeInternals", "source", "subscriber", "project", "concurrent", "onBeforeNext", "expand", "innerSubScheduler", "additionalFinalizer", "buffer", "active", "index", "isComplete", "checkComplete", "outerNext", "value", "doInnerSub", "innerComplete", "innerFrom", "createOperatorSubscriber", "innerValue", "bufferedValue", "executeSchedule", "err", "mergeMap", "project", "resultSelector", "concurrent", "isFunction", "a", "i", "map", "b", "ii", "innerFrom", "operate", "source", "subscriber", "mergeInternals", "mergeAll", "concurrent", "mergeMap", "identity", "concatAll", "mergeAll", "concat", "args", "_i", "concatAll", "from", "popScheduler", "defer", "observableFactory", "Observable", "subscriber", "innerFrom", "nodeEventEmitterMethods", "eventTargetMethods", "jqueryMethods", "fromEvent", "target", "eventName", "options", "resultSelector", "isFunction", "mapOneOrManyArgs", "_a", "__read", "isEventTarget", "methodName", "handler", "isNodeStyleEventEmitter", "toCommonHandlerRegistry", "isJQueryStyleEventEmitter", "add", "remove", "isArrayLike", "mergeMap", "subTarget", "innerFrom", "Observable", "subscriber", "args", "_i", "fromEventPattern", "addHandler", "removeHandler", "resultSelector", "mapOneOrManyArgs", "Observable", "subscriber", "handler", "e", "_i", "retValue", "isFunction", "timer", "dueTime", "intervalOrScheduler", "scheduler", "async", "intervalDuration", "isScheduler", "Observable", "subscriber", "due", "isValidDate", "n", "merge", "args", "_i", "scheduler", "popScheduler", "concurrent", "popNumber", "sources", "innerFrom", "mergeAll", "from", "EMPTY", "NEVER", "Observable", "noop", "isArray", "argsOrArgArray", "args", "filter", "predicate", "thisArg", "operate", "source", "subscriber", "index", "createOperatorSubscriber", "value", "zip", "args", "_i", "resultSelector", "popResultSelector", "sources", "argsOrArgArray", "Observable", "subscriber", "buffers", "completed", "sourceIndex", "innerFrom", "createOperatorSubscriber", "value", "buffer", "result", "__spreadArray", "__read", "i", "EMPTY", "audit", "durationSelector", "operate", "source", "subscriber", "hasValue", "lastValue", "durationSubscriber", "isComplete", "endDuration", "value", "cleanupDuration", "createOperatorSubscriber", "innerFrom", "auditTime", "duration", "scheduler", "asyncScheduler", "audit", "timer", "bufferCount", "bufferSize", "startBufferEvery", "operate", "source", "subscriber", "buffers", "count", "createOperatorSubscriber", "value", "toEmit", "buffers_1", "__values", "buffers_1_1", "buffer", "toEmit_1", "toEmit_1_1", "arrRemove", "buffers_2", "buffers_2_1", "catchError", "selector", "operate", "source", "subscriber", "innerSub", "syncUnsub", "handledResult", "createOperatorSubscriber", "err", "innerFrom", "scanInternals", "accumulator", "seed", "hasSeed", "emitOnNext", "emitBeforeComplete", "source", "subscriber", "hasState", "state", "index", "createOperatorSubscriber", "value", "i", "combineLatest", "args", "_i", "resultSelector", "popResultSelector", "pipe", "__spreadArray", "__read", "mapOneOrManyArgs", "operate", "source", "subscriber", "combineLatestInit", "argsOrArgArray", "combineLatestWith", "otherSources", "_i", "combineLatest", "__spreadArray", "__read", "debounceTime", "dueTime", "scheduler", "asyncScheduler", "operate", "source", "subscriber", "activeTask", "lastValue", "lastTime", "emit", "value", "emitWhenIdle", "targetTime", "now", "createOperatorSubscriber", "defaultIfEmpty", "defaultValue", "operate", "source", "subscriber", "hasValue", "createOperatorSubscriber", "value", "take", "count", "EMPTY", "operate", "source", "subscriber", "seen", "createOperatorSubscriber", "value", "ignoreElements", "operate", "source", "subscriber", "createOperatorSubscriber", "noop", "mapTo", "value", "map", "delayWhen", "delayDurationSelector", "subscriptionDelay", "source", "concat", "take", "ignoreElements", "mergeMap", "value", "index", "innerFrom", "mapTo", "delay", "due", "scheduler", "asyncScheduler", "duration", "timer", "delayWhen", "distinctUntilChanged", "comparator", "keySelector", "identity", "defaultCompare", "operate", "source", "subscriber", "previousKey", "first", "createOperatorSubscriber", "value", "currentKey", "a", "b", "distinctUntilKeyChanged", "key", "compare", "distinctUntilChanged", "x", "y", "throwIfEmpty", "errorFactory", "defaultErrorFactory", "operate", "source", "subscriber", "hasValue", "createOperatorSubscriber", "value", "EmptyError", "endWith", "values", "_i", "source", "concat", "of", "__spreadArray", "__read", "finalize", "callback", "operate", "source", "subscriber", "first", "predicate", "defaultValue", "hasDefaultValue", "source", "filter", "v", "identity", "take", "defaultIfEmpty", "throwIfEmpty", "EmptyError", "takeLast", "count", "EMPTY", "operate", "source", "subscriber", "buffer", "createOperatorSubscriber", "value", "buffer_1", "__values", "buffer_1_1", "merge", "args", "_i", "scheduler", "popScheduler", "concurrent", "popNumber", "argsOrArgArray", "operate", "source", "subscriber", "mergeAll", "from", "__spreadArray", "__read", "mergeWith", "otherSources", "_i", "merge", "__spreadArray", "__read", "repeat", "countOrConfig", "count", "delay", "_a", "EMPTY", "operate", "source", "subscriber", "soFar", "sourceSub", "resubscribe", "notifier", "timer", "innerFrom", "notifierSubscriber_1", "createOperatorSubscriber", "subscribeToSource", "syncUnsub", "scan", "accumulator", "seed", "operate", "scanInternals", "share", "options", "_a", "connector", "Subject", "_b", "resetOnError", "_c", "resetOnComplete", "_d", "resetOnRefCountZero", "wrapperSource", "connection", "resetConnection", "subject", "refCount", "hasCompleted", "hasErrored", "cancelReset", "reset", "resetAndUnsubscribe", "conn", "operate", "source", "subscriber", "dest", "handleReset", "SafeSubscriber", "value", "err", "innerFrom", "on", "args", "_i", "onSubscriber", "__spreadArray", "__read", "shareReplay", "configOrBufferSize", "windowTime", "scheduler", "bufferSize", "refCount", "_a", "_b", "_c", "share", "ReplaySubject", "skip", "count", "filter", "_", "index", "skipUntil", "notifier", "operate", "source", "subscriber", "taking", "skipSubscriber", "createOperatorSubscriber", "noop", "innerFrom", "value", "startWith", "values", "_i", "scheduler", "popScheduler", "operate", "source", "subscriber", "concat", "switchMap", "project", "resultSelector", "operate", "source", "subscriber", "innerSubscriber", "index", "isComplete", "checkComplete", "createOperatorSubscriber", "value", "innerIndex", "outerIndex", "innerFrom", "innerValue", "takeUntil", "notifier", "operate", "source", "subscriber", "innerFrom", "createOperatorSubscriber", "noop", "takeWhile", "predicate", "inclusive", "operate", "source", "subscriber", "index", "createOperatorSubscriber", "value", "result", "tap", "observerOrNext", "error", "complete", "tapObserver", "isFunction", "operate", "source", "subscriber", "_a", "isUnsub", "createOperatorSubscriber", "value", "err", "_b", "identity", "throttle", "durationSelector", "config", "operate", "source", "subscriber", "_a", "_b", "leading", "_c", "trailing", "hasValue", "sendValue", "throttled", "isComplete", "endThrottling", "send", "cleanupThrottling", "startThrottle", "value", "innerFrom", "createOperatorSubscriber", "throttleTime", "duration", "scheduler", "config", "asyncScheduler", "duration$", "timer", "throttle", "withLatestFrom", "inputs", "_i", "project", "popResultSelector", "operate", "source", "subscriber", "len", "otherValues", "hasValue", "ready", "i", "innerFrom", "createOperatorSubscriber", "value", "identity", "noop", "values", "__spreadArray", "__read", "zip", "sources", "_i", "operate", "source", "subscriber", "__spreadArray", "__read", "zipWith", "otherInputs", "_i", "zip", "__spreadArray", "__read", "watchDocument", "document$", "ReplaySubject", "fromEvent", "getElements", "selector", "node", "getElement", "el", "getOptionalElement", "getActiveElement", "_a", "_b", "_c", "_d", "observer$", "merge", "fromEvent", "debounceTime", "startWith", "map", "getActiveElement", "shareReplay", "watchElementFocus", "el", "active", "distinctUntilChanged", "watchElementHover", "el", "duration", "merge", "fromEvent", "map", "debounceTime", "identity", "startWith", "getElementOffset", "el", "watchElementOffset", "merge", "fromEvent", "auditTime", "animationFrameScheduler", "map", "startWith", "getElementContentOffset", "el", "watchElementContentOffset", "merge", "fromEvent", "auditTime", "animationFrameScheduler", "map", "startWith", "appendChild", "el", "child", "node", "h", "tag", "attributes", "children", "attr", "round", "value", "digits", "watchScript", "src", "script", "h", "defer", "merge", "fromEvent", "switchMap", "throwError", "map", "finalize", "take", "entry$", "Subject", "observer$", "defer", "watchScript", "of", "map", "entries", "entry", "switchMap", "observer", "merge", "NEVER", "finalize", "shareReplay", "getElementSize", "el", "watchElementSize", "tap", "filter", "target", "startWith", "getElementContentSize", "el", "getElementContainer", "parent", "entry$", "Subject", "observer$", "defer", "of", "entries", "entry", "switchMap", "observer", "merge", "NEVER", "finalize", "shareReplay", "watchElementVisibility", "el", "tap", "filter", "target", "map", "isIntersecting", "watchElementBoundary", "threshold", "watchElementContentOffset", "y", "visible", "getElementSize", "content", "getElementContentSize", "distinctUntilChanged", "toggles", "getElement", "getToggle", "name", "setToggle", "value", "watchToggle", "el", "fromEvent", "map", "startWith", "isSusceptibleToKeyboard", "el", "type", "watchComposition", "merge", "fromEvent", "map", "startWith", "watchKeyboard", "keyboard$", "filter", "ev", "getToggle", "mode", "active", "getActiveElement", "share", "switchMap", "EMPTY", "getLocation", "setLocation", "url", "navigate", "feature", "el", "h", "watchLocation", "Subject", "getLocationHash", "setLocationHash", "hash", "el", "h", "ev", "watchLocationHash", "location$", "merge", "fromEvent", "map", "startWith", "filter", "shareReplay", "watchLocationTarget", "id", "getOptionalElement", "watchMedia", "query", "media", "fromEventPattern", "next", "startWith", "watchPrint", "merge", "fromEvent", "map", "at", "query$", "factory", "switchMap", "active", "EMPTY", "request", "url", "options", "Observable", "observer", "req", "event", "_a", "length", "requestJSON", "switchMap", "res", "map", "body", "shareReplay", "requestHTML", "dom", "requestXML", "getViewportOffset", "watchViewportOffset", "merge", "fromEvent", "map", "startWith", "getViewportSize", "watchViewportSize", "fromEvent", "map", "startWith", "watchViewport", "combineLatest", "watchViewportOffset", "watchViewportSize", "map", "offset", "size", "shareReplay", "watchViewportAt", "el", "viewport$", "header$", "size$", "distinctUntilKeyChanged", "offset$", "combineLatest", "map", "getElementOffset", "height", "offset", "size", "x", "y", "recv", "worker", "fromEvent", "ev", "send", "send$", "Subject", "data", "watchWorker", "url", "recv$", "worker$", "done$", "ignoreElements", "endWith", "mergeWith", "takeUntil", "share", "script", "getElement", "config", "getLocation", "configuration", "feature", "flag", "translation", "key", "value", "getComponentElement", "type", "node", "getElement", "getComponentElements", "getElements", "watchAnnounce", "el", "button", "getElement", "fromEvent", "map", "content", "mountAnnounce", "feature", "EMPTY", "defer", "push$", "Subject", "hash", "tap", "state", "finalize", "__spreadValues", "watchConsent", "el", "target$", "map", "target", "mountConsent", "options", "internal$", "Subject", "hidden", "tap", "state", "finalize", "__spreadValues", "renderTooltip", "id", "style", "h", "renderAnnotation", "id", "prefix", "anchor", "h", "renderTooltip", "renderClipboardButton", "id", "h", "translation", "renderSearchDocument", "document", "flag", "parent", "teaser", "missing", "key", "list", "h", "config", "configuration", "url", "feature", "match", "highlight", "value", "tags", "tag", "type", "translation", "renderSearchResultItem", "result", "threshold", "docs", "doc", "article", "index", "best", "more", "children", "section", "renderSourceFacts", "facts", "h", "key", "value", "round", "renderTabbedControl", "type", "classes", "h", "renderTable", "table", "h", "renderVersion", "version", "config", "configuration", "url", "h", "renderVersionSelector", "versions", "active", "translation", "sequence", "watchTooltip", "el", "host", "width", "getElementSize", "container", "getElementContainer", "scroll$", "watchElementContentOffset", "of", "active$", "merge", "watchElementFocus", "watchElementHover", "distinctUntilChanged", "combineLatest", "map", "active", "scroll", "x", "y", "getElementOffset", "size", "table", "mountTooltip", "title", "EMPTY", "id", "tooltip", "renderTooltip", "typeset", "getElement", "defer", "push$", "Subject", "offset", "filter", "debounceTime", "auditTime", "animationFrameScheduler", "throttleTime", "origin", "tap", "state", "finalize", "__spreadValues", "subscribeOn", "asyncScheduler", "watchAnnotation", "el", "container", "offset$", "defer", "combineLatest", "watchElementOffset", "watchElementContentOffset", "map", "x", "y", "scroll", "width", "height", "getElementSize", "watchElementFocus", "switchMap", "active", "offset", "take", "mountAnnotation", "target$", "tooltip", "index", "push$", "Subject", "done$", "ignoreElements", "endWith", "watchElementVisibility", "takeUntil", "visible", "merge", "filter", "debounceTime", "auditTime", "animationFrameScheduler", "throttleTime", "origin", "fromEvent", "ev", "withLatestFrom", "_a", "parent", "getActiveElement", "target", "delay", "tap", "state", "finalize", "__spreadValues", "findHosts", "container", "getElements", "findMarkers", "markers", "el", "nodes", "it", "node", "text", "match", "id", "force", "marker", "swap", "source", "target", "mountAnnotationList", "target$", "print$", "parent", "prefix", "annotations", "getOptionalElement", "renderAnnotation", "EMPTY", "defer", "push$", "Subject", "done$", "ignoreElements", "endWith", "pairs", "annotation", "getElement", "takeUntil", "active", "inner", "child", "merge", "mountAnnotation", "finalize", "share", "findList", "el", "sibling", "mountAnnotationBlock", "options", "defer", "list", "mountAnnotationList", "EMPTY", "import_clipboard", "sequence", "findCandidateList", "el", "sibling", "watchCodeBlock", "watchElementSize", "map", "width", "getElementContentSize", "distinctUntilKeyChanged", "mountCodeBlock", "options", "hover", "factory$", "defer", "push$", "Subject", "done$", "takeLast", "scrollable", "content$", "ClipboardJS", "feature", "parent", "button", "renderClipboardButton", "mountTooltip", "container", "list", "annotations$", "mountAnnotationList", "takeUntil", "height", "distinctUntilChanged", "switchMap", "active", "EMPTY", "tap", "state", "finalize", "__spreadValues", "mergeWith", "watchElementVisibility", "filter", "visible", "take", "watchDetails", "el", "target$", "print$", "open", "merge", "map", "target", "filter", "details", "active", "tap", "mountDetails", "options", "defer", "push$", "Subject", "action", "reveal", "state", "finalize", "__spreadValues", "mermaid_default", "mermaid$", "sequence", "fetchScripts", "watchScript", "of", "mountMermaid", "el", "tap", "mermaid_default", "map", "shareReplay", "__async", "id", "host", "h", "text", "svg", "fn", "shadow", "sentinel", "h", "mountDataTable", "el", "renderTable", "of", "watchContentTabs", "inputs", "initial", "input", "merge", "fromEvent", "map", "getElement", "startWith", "active", "mountContentTabs", "el", "viewport$", "target$", "container", "getElements", "prev", "renderTabbedControl", "next", "defer", "push$", "Subject", "done$", "ignoreElements", "endWith", "combineLatest", "watchElementSize", "takeUntil", "auditTime", "animationFrameScheduler", "size", "offset", "getElementOffset", "width", "getElementSize", "content", "getElementContentOffset", "watchElementContentOffset", "getElementContentSize", "direction", "filter", "label", "h", "ev", "tap", "feature", "skip", "withLatestFrom", "tab", "y", "set", "tabs", "media", "state", "finalize", "__spreadValues", "subscribeOn", "asyncScheduler", "mountContent", "el", "viewport$", "target$", "print$", "merge", "getElements", "child", "mountAnnotationBlock", "mountCodeBlock", "mountMermaid", "mountDataTable", "mountDetails", "mountContentTabs", "feature", "mountTooltip", "watchDialog", "_el", "alert$", "switchMap", "message", "merge", "of", "delay", "map", "active", "mountDialog", "el", "options", "inner", "getElement", "defer", "push$", "Subject", "tap", "state", "finalize", "__spreadValues", "isHidden", "viewport$", "feature", "of", "direction$", "map", "y", "bufferCount", "a", "b", "distinctUntilKeyChanged", "hidden$", "combineLatest", "filter", "offset", "direction", "distinctUntilChanged", "search$", "watchToggle", "search", "switchMap", "active", "startWith", "watchHeader", "el", "options", "defer", "watchElementSize", "height", "hidden", "shareReplay", "mountHeader", "header$", "main$", "push$", "Subject", "done$", "ignoreElements", "endWith", "combineLatestWith", "tooltips", "from", "getElements", "mergeMap", "child", "mountTooltip", "takeUntil", "state", "__spreadValues", "mergeWith", "watchHeaderTitle", "el", "viewport$", "header$", "watchViewportAt", "map", "y", "height", "getElementSize", "distinctUntilKeyChanged", "mountHeaderTitle", "options", "defer", "push$", "Subject", "active", "heading", "getOptionalElement", "EMPTY", "tap", "state", "finalize", "__spreadValues", "watchMain", "el", "viewport$", "header$", "adjust$", "map", "height", "distinctUntilChanged", "border$", "switchMap", "watchElementSize", "distinctUntilKeyChanged", "combineLatest", "header", "top", "bottom", "y", "a", "b", "watchPalette", "inputs", "current", "input", "index", "of", "mergeMap", "fromEvent", "map", "startWith", "shareReplay", "mountPalette", "el", "getElements", "meta", "h", "scheme", "media$", "watchMedia", "defer", "push$", "Subject", "palette", "media", "key", "value", "label", "header", "getComponentElement", "style", "color", "observeOn", "asyncScheduler", "takeUntil", "skip", "repeat", "tap", "state", "finalize", "__spreadValues", "mountProgress", "el", "progress$", "defer", "push$", "Subject", "value", "tap", "finalize", "map", "import_clipboard", "extract", "el", "copy", "text", "setupClipboardJS", "alert$", "ClipboardJS", "Observable", "subscriber", "getElement", "ev", "tap", "map", "translation", "resolve", "url", "base", "extract", "document", "sitemap", "el", "getElements", "getElement", "links", "link", "href", "fetchSitemap", "requestXML", "map", "catchError", "of", "handle", "ev", "sitemap", "EMPTY", "el", "url", "of", "head", "document", "tags", "getElements", "resolve", "key", "value", "inject", "next", "selector", "feature", "source", "getOptionalElement", "target", "html", "name", "container", "getComponentElement", "concat", "switchMap", "script", "Observable", "observer", "ignoreElements", "endWith", "setupInstantNavigation", "location$", "viewport$", "progress$", "config", "configuration", "sitemap$", "fetchSitemap", "instant$", "fromEvent", "combineLatestWith", "share", "history$", "map", "getLocation", "withLatestFrom", "offset", "merge", "document$", "distinctUntilKeyChanged", "requestHTML", "catchError", "setLocation", "_", "distinctUntilChanged", "a", "b", "tap", "_a", "_b", "setLocationHash", "debounceTime", "import_escape_html", "setupSearchHighlighter", "config", "regex", "term", "separator", "highlight", "_", "data", "query", "match", "value", "escapeHTML", "isSearchReadyMessage", "message", "isSearchResultMessage", "setupSearchWorker", "url", "index$", "worker$", "watchWorker", "merge", "of", "watchToggle", "first", "active", "switchMap", "config", "docs", "feature", "setupVersionSelector", "document$", "config", "configuration", "versions$", "requestJSON", "catchError", "EMPTY", "current$", "map", "versions", "current", "version", "aliases", "switchMap", "urls", "fromEvent", "filter", "ev", "withLatestFrom", "el", "url", "of", "fetchSitemap", "sitemap", "path", "getLocation", "setLocation", "combineLatest", "getElement", "renderVersionSelector", "_a", "outdated", "ignored", "main", "ignore", "warning", "getComponentElements", "watchSearchQuery", "el", "worker$", "searchParams", "getLocation", "setToggle", "watchToggle", "first", "active", "url", "focus$", "watchElementFocus", "value$", "merge", "isSearchReadyMessage", "fromEvent", "map", "distinctUntilChanged", "combineLatest", "value", "focus", "shareReplay", "mountSearchQuery", "push$", "Subject", "done$", "ignoreElements", "endWith", "_", "query", "distinctUntilKeyChanged", "takeUntil", "label", "getElement", "tap", "state", "finalize", "__spreadValues", "mountSearchResult", "el", "worker$", "query$", "push$", "Subject", "boundary$", "watchElementBoundary", "filter", "container", "meta", "getElement", "list", "watchToggle", "active", "withLatestFrom", "skipUntil", "first", "isSearchReadyMessage", "items", "value", "translation", "count", "round", "render$", "tap", "switchMap", "merge", "of", "bufferCount", "zipWith", "chunk", "map", "renderSearchResultItem", "share", "item", "mergeMap", "details", "getOptionalElement", "EMPTY", "fromEvent", "takeUntil", "isSearchResultMessage", "data", "state", "finalize", "__spreadValues", "watchSearchShare", "_el", "query$", "map", "value", "url", "getLocation", "mountSearchShare", "el", "options", "push$", "Subject", "done$", "ignoreElements", "endWith", "fromEvent", "takeUntil", "ev", "tap", "state", "finalize", "__spreadValues", "mountSearchSuggest", "el", "worker$", "keyboard$", "push$", "Subject", "query", "getComponentElement", "query$", "merge", "fromEvent", "observeOn", "asyncScheduler", "map", "distinctUntilChanged", "combineLatestWith", "suggest", "value", "words", "last", "filter", "mode", "key", "isSearchResultMessage", "data", "tap", "state", "finalize", "mountSearch", "el", "index$", "keyboard$", "config", "configuration", "worker$", "setupSearchWorker", "query", "getComponentElement", "result", "fromEvent", "filter", "target", "setToggle", "mode", "key", "active", "getActiveElement", "anchors", "anchor", "getElements", "article", "best", "a", "b", "els", "i", "query$", "mountSearchQuery", "merge", "mountSearchResult", "mergeWith", "getComponentElements", "child", "mountSearchShare", "mountSearchSuggest", "err", "NEVER", "mountSearchHiglight", "el", "index$", "location$", "combineLatest", "startWith", "getLocation", "filter", "url", "map", "index", "setupSearchHighlighter", "fn", "_a", "nodes", "it", "node", "original", "replaced", "text", "childNodes", "h", "watchSidebar", "el", "viewport$", "main$", "parent", "adjust", "combineLatest", "map", "offset", "height", "y", "distinctUntilChanged", "a", "b", "mountSidebar", "_a", "_b", "header$", "options", "__objRest", "inner", "getElement", "getElementOffset", "defer", "push$", "Subject", "done$", "ignoreElements", "endWith", "next$", "auditTime", "animationFrameScheduler", "withLatestFrom", "first", "item", "getElements", "container", "getElementSize", "from", "mergeMap", "label", "fromEvent", "observeOn", "asyncScheduler", "takeUntil", "input", "tap", "state", "finalize", "__spreadValues", "fetchSourceFactsFromGitHub", "user", "repo", "url", "zip", "requestJSON", "catchError", "EMPTY", "map", "release", "defaultIfEmpty", "info", "__spreadValues", "fetchSourceFactsFromGitLab", "base", "project", "url", "requestJSON", "catchError", "EMPTY", "map", "star_count", "forks_count", "defaultIfEmpty", "fetchSourceFacts", "url", "match", "user", "repo", "fetchSourceFactsFromGitHub", "base", "slug", "fetchSourceFactsFromGitLab", "EMPTY", "fetch$", "watchSource", "el", "defer", "cached", "of", "getComponentElements", "consent", "EMPTY", "fetchSourceFacts", "tap", "facts", "catchError", "filter", "map", "shareReplay", "mountSource", "inner", "getElement", "push$", "Subject", "renderSourceFacts", "state", "finalize", "__spreadValues", "watchTabs", "el", "viewport$", "header$", "watchElementSize", "switchMap", "watchViewportAt", "map", "y", "distinctUntilKeyChanged", "mountTabs", "options", "defer", "push$", "Subject", "hidden", "feature", "of", "tap", "state", "finalize", "__spreadValues", "watchTableOfContents", "el", "viewport$", "header$", "table", "anchors", "getElements", "anchor", "id", "target", "getOptionalElement", "adjust$", "distinctUntilKeyChanged", "map", "height", "main", "getComponentElement", "grid", "getElement", "share", "watchElementSize", "switchMap", "body", "defer", "path", "of", "index", "offset", "parent", "a", "b", "combineLatestWith", "adjust", "scan", "prev", "next", "y", "size", "last", "distinctUntilChanged", "startWith", "bufferCount", "mountTableOfContents", "main$", "target$", "push$", "Subject", "done$", "ignoreElements", "endWith", "feature", "smooth$", "merge", "debounceTime", "filter", "observeOn", "asyncScheduler", "withLatestFrom", "behavior", "container", "getElementContainer", "getElementSize", "takeUntil", "skip", "repeat", "url", "getLocation", "active", "hash", "tap", "state", "finalize", "__spreadValues", "watchBackToTop", "_el", "viewport$", "main$", "target$", "direction$", "map", "y", "bufferCount", "a", "b", "distinctUntilChanged", "active$", "active", "combineLatest", "direction", "takeUntil", "skip", "endWith", "repeat", "hidden", "mountBackToTop", "el", "header$", "push$", "Subject", "done$", "ignoreElements", "distinctUntilKeyChanged", "height", "fromEvent", "ev", "tap", "state", "finalize", "__spreadValues", "patchEllipsis", "document$", "switchMap", "getElements", "mergeMap", "el", "watchElementVisibility", "takeUntil", "skip", "filter", "visible", "map", "take", "text", "host", "mountTooltip", "finalize", "patchIndeterminate", "document$", "tablet$", "switchMap", "getElements", "tap", "el", "mergeMap", "fromEvent", "takeWhile", "map", "withLatestFrom", "tablet", "isAppleDevice", "patchScrollfix", "document$", "switchMap", "getElements", "tap", "el", "filter", "mergeMap", "fromEvent", "map", "top", "patchScrolllock", "viewport$", "tablet$", "combineLatest", "watchToggle", "map", "active", "tablet", "switchMap", "of", "delay", "withLatestFrom", "y", "value", "obj", "data", "key", "x", "y", "nodes", "parent", "i", "node", "fetchSearchIndex", "watchScript", "config", "map", "shareReplay", "requestJSON", "document$", "watchDocument", "location$", "watchLocation", "target$", "watchLocationTarget", "keyboard$", "watchKeyboard", "viewport$", "watchViewport", "tablet$", "watchMedia", "screen$", "print$", "watchPrint", "configuration", "index$", "NEVER", "alert$", "Subject", "setupClipboardJS", "progress$", "feature", "setupInstantNavigation", "_a", "setupVersionSelector", "merge", "delay", "setToggle", "filter", "mode", "key", "prev", "getOptionalElement", "setLocation", "next", "active", "getActiveElement", "patchEllipsis", "patchIndeterminate", "patchScrollfix", "patchScrolllock", "header$", "watchHeader", "getComponentElement", "main$", "switchMap", "el", "watchMain", "control$", "getComponentElements", "mountConsent", "mountDialog", "mountHeader", "mountPalette", "mountProgress", "mountSearch", "mountSource", "content$", "defer", "mountAnnounce", "mountContent", "mountSearchHiglight", "EMPTY", "mountHeaderTitle", "at", "mountSidebar", "mountTabs", "mountTableOfContents", "mountBackToTop", "component$", "mergeWith"]
+}
diff --git a/assets/javascripts/bundle.febc23d1.min.js b/assets/javascripts/bundle.febc23d1.min.js
deleted file mode 100644
index 2cb79c7c2..000000000
--- a/assets/javascripts/bundle.febc23d1.min.js
+++ /dev/null
@@ -1,29 +0,0 @@
-(()=>{var ta=Object.create;var yt=Object.defineProperty;var ra=Object.getOwnPropertyDescriptor;var oa=Object.getOwnPropertyNames,St=Object.getOwnPropertySymbols,na=Object.getPrototypeOf,ir=Object.prototype.hasOwnProperty,Qr=Object.prototype.propertyIsEnumerable;var Kr=(e,t,r)=>t in e?yt(e,t,{enumerable:!0,configurable:!0,writable:!0,value:r}):e[t]=r,I=(e,t)=>{for(var r in t||(t={}))ir.call(t,r)&&Kr(e,r,t[r]);if(St)for(var r of St(t))Qr.call(t,r)&&Kr(e,r,t[r]);return e};var ia=e=>yt(e,"__esModule",{value:!0});var Br=(e,t)=>{var r={};for(var o in e)ir.call(e,o)&&t.indexOf(o)<0&&(r[o]=e[o]);if(e!=null&&St)for(var o of St(e))t.indexOf(o)<0&&Qr.call(e,o)&&(r[o]=e[o]);return r};var wt=(e,t)=>()=>(t||e((t={exports:{}}).exports,t),t.exports);var aa=(e,t,r)=>{if(t&&typeof t=="object"||typeof t=="function")for(let o of oa(t))!ir.call(e,o)&&o!=="default"&&yt(e,o,{get:()=>t[o],enumerable:!(r=ra(t,o))||r.enumerable});return e},Qe=e=>aa(ia(yt(e!=null?ta(na(e)):{},"default",e&&e.__esModule&&"default"in e?{get:()=>e.default,enumerable:!0}:{value:e,enumerable:!0})),e);var Yr=wt((ar,Jr)=>{(function(e,t){typeof ar=="object"&&typeof Jr!="undefined"?t():typeof define=="function"&&define.amd?define(t):t()})(ar,function(){"use strict";function e(r){var o=!0,n=!1,i=null,a={text:!0,search:!0,url:!0,tel:!0,email:!0,password:!0,number:!0,date:!0,month:!0,week:!0,time:!0,datetime:!0,"datetime-local":!0};function s(w){return!!(w&&w!==document&&w.nodeName!=="HTML"&&w.nodeName!=="BODY"&&"classList"in w&&"contains"in w.classList)}function c(w){var $e=w.type,Ee=w.tagName;return!!(Ee==="INPUT"&&a[$e]&&!w.readOnly||Ee==="TEXTAREA"&&!w.readOnly||w.isContentEditable)}function l(w){w.classList.contains("focus-visible")||(w.classList.add("focus-visible"),w.setAttribute("data-focus-visible-added",""))}function p(w){!w.hasAttribute("data-focus-visible-added")||(w.classList.remove("focus-visible"),w.removeAttribute("data-focus-visible-added"))}function m(w){w.metaKey||w.altKey||w.ctrlKey||(s(r.activeElement)&&l(r.activeElement),o=!0)}function f(w){o=!1}function d(w){!s(w.target)||(o||c(w.target))&&l(w.target)}function v(w){!s(w.target)||(w.target.classList.contains("focus-visible")||w.target.hasAttribute("data-focus-visible-added"))&&(n=!0,window.clearTimeout(i),i=window.setTimeout(function(){n=!1},100),p(w.target))}function h(w){document.visibilityState==="hidden"&&(n&&(o=!0),$())}function $(){document.addEventListener("mousemove",j),document.addEventListener("mousedown",j),document.addEventListener("mouseup",j),document.addEventListener("pointermove",j),document.addEventListener("pointerdown",j),document.addEventListener("pointerup",j),document.addEventListener("touchmove",j),document.addEventListener("touchstart",j),document.addEventListener("touchend",j)}function K(){document.removeEventListener("mousemove",j),document.removeEventListener("mousedown",j),document.removeEventListener("mouseup",j),document.removeEventListener("pointermove",j),document.removeEventListener("pointerdown",j),document.removeEventListener("pointerup",j),document.removeEventListener("touchmove",j),document.removeEventListener("touchstart",j),document.removeEventListener("touchend",j)}function j(w){w.target.nodeName&&w.target.nodeName.toLowerCase()==="html"||(o=!1,K())}document.addEventListener("keydown",m,!0),document.addEventListener("mousedown",f,!0),document.addEventListener("pointerdown",f,!0),document.addEventListener("touchstart",f,!0),document.addEventListener("visibilitychange",h,!0),$(),r.addEventListener("focus",d,!0),r.addEventListener("blur",v,!0),r.nodeType===Node.DOCUMENT_FRAGMENT_NODE&&r.host?r.host.setAttribute("data-js-focus-visible",""):r.nodeType===Node.DOCUMENT_NODE&&(document.documentElement.classList.add("js-focus-visible"),document.documentElement.setAttribute("data-js-focus-visible",""))}if(typeof window!="undefined"&&typeof document!="undefined"){window.applyFocusVisiblePolyfill=e;var t;try{t=new CustomEvent("focus-visible-polyfill-ready")}catch(r){t=document.createEvent("CustomEvent"),t.initCustomEvent("focus-visible-polyfill-ready",!1,!1,{})}window.dispatchEvent(t)}typeof document!="undefined"&&e(document)})});var xo=wt((ls,Ot)=>{/*! *****************************************************************************
-Copyright (c) Microsoft Corporation.
-
-Permission to use, copy, modify, and/or distribute this software for any
-purpose with or without fee is hereby granted.
-
-THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
-REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
-AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
-INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
-OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-PERFORMANCE OF THIS SOFTWARE.
-***************************************************************************** */var Gr,Xr,Zr,eo,to,ro,oo,no,io,Et,sr,ao,so,co,Ke,lo,po,uo,fo,mo,ho,bo,vo,Tt;(function(e){var t=typeof global=="object"?global:typeof self=="object"?self:typeof this=="object"?this:{};typeof define=="function"&&define.amd?define("tslib",["exports"],function(o){e(r(t,r(o)))}):typeof Ot=="object"&&typeof Ot.exports=="object"?e(r(t,r(Ot.exports))):e(r(t));function r(o,n){return o!==t&&(typeof Object.create=="function"?Object.defineProperty(o,"__esModule",{value:!0}):o.__esModule=!0),function(i,a){return o[i]=n?n(i,a):a}}})(function(e){var t=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(o,n){o.__proto__=n}||function(o,n){for(var i in n)Object.prototype.hasOwnProperty.call(n,i)&&(o[i]=n[i])};Gr=function(o,n){if(typeof n!="function"&&n!==null)throw new TypeError("Class extends value "+String(n)+" is not a constructor or null");t(o,n);function i(){this.constructor=o}o.prototype=n===null?Object.create(n):(i.prototype=n.prototype,new i)},Xr=Object.assign||function(o){for(var n,i=1,a=arguments.length;i<a;i++){n=arguments[i];for(var s in n)Object.prototype.hasOwnProperty.call(n,s)&&(o[s]=n[s])}return o},Zr=function(o,n){var i={};for(var a in o)Object.prototype.hasOwnProperty.call(o,a)&&n.indexOf(a)<0&&(i[a]=o[a]);if(o!=null&&typeof Object.getOwnPropertySymbols=="function")for(var s=0,a=Object.getOwnPropertySymbols(o);s<a.length;s++)n.indexOf(a[s])<0&&Object.prototype.propertyIsEnumerable.call(o,a[s])&&(i[a[s]]=o[a[s]]);return i},eo=function(o,n,i,a){var s=arguments.length,c=s<3?n:a===null?a=Object.getOwnPropertyDescriptor(n,i):a,l;if(typeof Reflect=="object"&&typeof Reflect.decorate=="function")c=Reflect.decorate(o,n,i,a);else for(var p=o.length-1;p>=0;p--)(l=o[p])&&(c=(s<3?l(c):s>3?l(n,i,c):l(n,i))||c);return s>3&&c&&Object.defineProperty(n,i,c),c},to=function(o,n){return function(i,a){n(i,a,o)}},ro=function(o,n){if(typeof Reflect=="object"&&typeof Reflect.metadata=="function")return Reflect.metadata(o,n)},oo=function(o,n,i,a){function s(c){return c instanceof i?c:new i(function(l){l(c)})}return new(i||(i=Promise))(function(c,l){function p(d){try{f(a.next(d))}catch(v){l(v)}}function m(d){try{f(a.throw(d))}catch(v){l(v)}}function f(d){d.done?c(d.value):s(d.value).then(p,m)}f((a=a.apply(o,n||[])).next())})},no=function(o,n){var i={label:0,sent:function(){if(c[0]&1)throw c[1];return c[1]},trys:[],ops:[]},a,s,c,l;return l={next:p(0),throw:p(1),return:p(2)},typeof Symbol=="function"&&(l[Symbol.iterator]=function(){return this}),l;function p(f){return function(d){return m([f,d])}}function m(f){if(a)throw new TypeError("Generator is already executing.");for(;i;)try{if(a=1,s&&(c=f[0]&2?s.return:f[0]?s.throw||((c=s.return)&&c.call(s),0):s.next)&&!(c=c.call(s,f[1])).done)return c;switch(s=0,c&&(f=[f[0]&2,c.value]),f[0]){case 0:case 1:c=f;break;case 4:return i.label++,{value:f[1],done:!1};case 5:i.label++,s=f[1],f=[0];continue;case 7:f=i.ops.pop(),i.trys.pop();continue;default:if(c=i.trys,!(c=c.length>0&&c[c.length-1])&&(f[0]===6||f[0]===2)){i=0;continue}if(f[0]===3&&(!c||f[1]>c[0]&&f[1]<c[3])){i.label=f[1];break}if(f[0]===6&&i.label<c[1]){i.label=c[1],c=f;break}if(c&&i.label<c[2]){i.label=c[2],i.ops.push(f);break}c[2]&&i.ops.pop(),i.trys.pop();continue}f=n.call(o,i)}catch(d){f=[6,d],s=0}finally{a=c=0}if(f[0]&5)throw f[1];return{value:f[0]?f[1]:void 0,done:!0}}},io=function(o,n){for(var i in o)i!=="default"&&!Object.prototype.hasOwnProperty.call(n,i)&&Tt(n,o,i)},Tt=Object.create?function(o,n,i,a){a===void 0&&(a=i),Object.defineProperty(o,a,{enumerable:!0,get:function(){return n[i]}})}:function(o,n,i,a){a===void 0&&(a=i),o[a]=n[i]},Et=function(o){var n=typeof Symbol=="function"&&Symbol.iterator,i=n&&o[n],a=0;if(i)return i.call(o);if(o&&typeof o.length=="number")return{next:function(){return o&&a>=o.length&&(o=void 0),{value:o&&o[a++],done:!o}}};throw new TypeError(n?"Object is not iterable.":"Symbol.iterator is not defined.")},sr=function(o,n){var i=typeof Symbol=="function"&&o[Symbol.iterator];if(!i)return o;var a=i.call(o),s,c=[],l;try{for(;(n===void 0||n-- >0)&&!(s=a.next()).done;)c.push(s.value)}catch(p){l={error:p}}finally{try{s&&!s.done&&(i=a.return)&&i.call(a)}finally{if(l)throw l.error}}return c},ao=function(){for(var o=[],n=0;n<arguments.length;n++)o=o.concat(sr(arguments[n]));return o},so=function(){for(var o=0,n=0,i=arguments.length;n<i;n++)o+=arguments[n].length;for(var a=Array(o),s=0,n=0;n<i;n++)for(var c=arguments[n],l=0,p=c.length;l<p;l++,s++)a[s]=c[l];return a},co=function(o,n){for(var i=0,a=n.length,s=o.length;i<a;i++,s++)o[s]=n[i];return o},Ke=function(o){return this instanceof Ke?(this.v=o,this):new Ke(o)},lo=function(o,n,i){if(!Symbol.asyncIterator)throw new TypeError("Symbol.asyncIterator is not defined.");var a=i.apply(o,n||[]),s,c=[];return s={},l("next"),l("throw"),l("return"),s[Symbol.asyncIterator]=function(){return this},s;function l(h){a[h]&&(s[h]=function($){return new Promise(function(K,j){c.push([h,$,K,j])>1||p(h,$)})})}function p(h,$){try{m(a[h]($))}catch(K){v(c[0][3],K)}}function m(h){h.value instanceof Ke?Promise.resolve(h.value.v).then(f,d):v(c[0][2],h)}function f(h){p("next",h)}function d(h){p("throw",h)}function v(h,$){h($),c.shift(),c.length&&p(c[0][0],c[0][1])}},po=function(o){var n,i;return n={},a("next"),a("throw",function(s){throw s}),a("return"),n[Symbol.iterator]=function(){return this},n;function a(s,c){n[s]=o[s]?function(l){return(i=!i)?{value:Ke(o[s](l)),done:s==="return"}:c?c(l):l}:c}},uo=function(o){if(!Symbol.asyncIterator)throw new TypeError("Symbol.asyncIterator is not defined.");var n=o[Symbol.asyncIterator],i;return n?n.call(o):(o=typeof Et=="function"?Et(o):o[Symbol.iterator](),i={},a("next"),a("throw"),a("return"),i[Symbol.asyncIterator]=function(){return this},i);function a(c){i[c]=o[c]&&function(l){return new Promise(function(p,m){l=o[c](l),s(p,m,l.done,l.value)})}}function s(c,l,p,m){Promise.resolve(m).then(function(f){c({value:f,done:p})},l)}},fo=function(o,n){return Object.defineProperty?Object.defineProperty(o,"raw",{value:n}):o.raw=n,o};var r=Object.create?function(o,n){Object.defineProperty(o,"default",{enumerable:!0,value:n})}:function(o,n){o.default=n};mo=function(o){if(o&&o.__esModule)return o;var n={};if(o!=null)for(var i in o)i!=="default"&&Object.prototype.hasOwnProperty.call(o,i)&&Tt(n,o,i);return r(n,o),n},ho=function(o){return o&&o.__esModule?o:{default:o}},bo=function(o,n){if(!n.has(o))throw new TypeError("attempted to get private field on non-instance");return n.get(o)},vo=function(o,n,i){if(!n.has(o))throw new TypeError("attempted to set private field on non-instance");return n.set(o,i),i},e("__extends",Gr),e("__assign",Xr),e("__rest",Zr),e("__decorate",eo),e("__param",to),e("__metadata",ro),e("__awaiter",oo),e("__generator",no),e("__exportStar",io),e("__createBinding",Tt),e("__values",Et),e("__read",sr),e("__spread",ao),e("__spreadArrays",so),e("__spreadArray",co),e("__await",Ke),e("__asyncGenerator",lo),e("__asyncDelegator",po),e("__asyncValues",uo),e("__makeTemplateObject",fo),e("__importStar",mo),e("__importDefault",ho),e("__classPrivateFieldGet",bo),e("__classPrivateFieldSet",vo)})});var Rr=wt((xt,jr)=>{/*!
- * clipboard.js v2.0.8
- * https://clipboardjs.com/
- *
- * Licensed MIT © Zeno Rocha
- */(function(t,r){typeof xt=="object"&&typeof jr=="object"?jr.exports=r():typeof define=="function"&&define.amd?define([],r):typeof xt=="object"?xt.ClipboardJS=r():t.ClipboardJS=r()})(xt,function(){return function(){var e={134:function(o,n,i){"use strict";i.d(n,{default:function(){return Zi}});var a=i(279),s=i.n(a),c=i(370),l=i.n(c),p=i(817),m=i.n(p);function f(_){return typeof Symbol=="function"&&typeof Symbol.iterator=="symbol"?f=function(b){return typeof b}:f=function(b){return b&&typeof Symbol=="function"&&b.constructor===Symbol&&b!==Symbol.prototype?"symbol":typeof b},f(_)}function d(_,x){if(!(_ instanceof x))throw new TypeError("Cannot call a class as a function")}function v(_,x){for(var b=0;b<x.length;b++){var k=x[b];k.enumerable=k.enumerable||!1,k.configurable=!0,"value"in k&&(k.writable=!0),Object.defineProperty(_,k.key,k)}}function h(_,x,b){return x&&v(_.prototype,x),b&&v(_,b),_}var $=function(){function _(x){d(this,_),this.resolveOptions(x),this.initSelection()}return h(_,[{key:"resolveOptions",value:function(){var b=arguments.length>0&&arguments[0]!==void 0?arguments[0]:{};this.action=b.action,this.container=b.container,this.emitter=b.emitter,this.target=b.target,this.text=b.text,this.trigger=b.trigger,this.selectedText=""}},{key:"initSelection",value:function(){this.text?this.selectFake():this.target&&this.selectTarget()}},{key:"createFakeElement",value:function(){var b=document.documentElement.getAttribute("dir")==="rtl";this.fakeElem=document.createElement("textarea"),this.fakeElem.style.fontSize="12pt",this.fakeElem.style.border="0",this.fakeElem.style.padding="0",this.fakeElem.style.margin="0",this.fakeElem.style.position="absolute",this.fakeElem.style[b?"right":"left"]="-9999px";var k=window.pageYOffset||document.documentElement.scrollTop;return this.fakeElem.style.top="".concat(k,"px"),this.fakeElem.setAttribute("readonly",""),this.fakeElem.value=this.text,this.fakeElem}},{key:"selectFake",value:function(){var b=this,k=this.createFakeElement();this.fakeHandlerCallback=function(){return b.removeFake()},this.fakeHandler=this.container.addEventListener("click",this.fakeHandlerCallback)||!0,this.container.appendChild(k),this.selectedText=m()(k),this.copyText(),this.removeFake()}},{key:"removeFake",value:function(){this.fakeHandler&&(this.container.removeEventListener("click",this.fakeHandlerCallback),this.fakeHandler=null,this.fakeHandlerCallback=null),this.fakeElem&&(this.container.removeChild(this.fakeElem),this.fakeElem=null)}},{key:"selectTarget",value:function(){this.selectedText=m()(this.target),this.copyText()}},{key:"copyText",value:function(){var b;try{b=document.execCommand(this.action)}catch(k){b=!1}this.handleResult(b)}},{key:"handleResult",value:function(b){this.emitter.emit(b?"success":"error",{action:this.action,text:this.selectedText,trigger:this.trigger,clearSelection:this.clearSelection.bind(this)})}},{key:"clearSelection",value:function(){this.trigger&&this.trigger.focus(),document.activeElement.blur(),window.getSelection().removeAllRanges()}},{key:"destroy",value:function(){this.removeFake()}},{key:"action",set:function(){var b=arguments.length>0&&arguments[0]!==void 0?arguments[0]:"copy";if(this._action=b,this._action!=="copy"&&this._action!=="cut")throw new Error('Invalid "action" value, use either "copy" or "cut"')},get:function(){return this._action}},{key:"target",set:function(b){if(b!==void 0)if(b&&f(b)==="object"&&b.nodeType===1){if(this.action==="copy"&&b.hasAttribute("disabled"))throw new Error('Invalid "target" attribute. Please use "readonly" instead of "disabled" attribute');if(this.action==="cut"&&(b.hasAttribute("readonly")||b.hasAttribute("disabled")))throw new Error(`Invalid "target" attribute. You can't cut text from elements with "readonly" or "disabled" attributes`);this._target=b}else throw new Error('Invalid "target" value, use a valid Element')},get:function(){return this._target}}]),_}(),K=$;function j(_){return typeof Symbol=="function"&&typeof Symbol.iterator=="symbol"?j=function(b){return typeof b}:j=function(b){return b&&typeof Symbol=="function"&&b.constructor===Symbol&&b!==Symbol.prototype?"symbol":typeof b},j(_)}function w(_,x){if(!(_ instanceof x))throw new TypeError("Cannot call a class as a function")}function $e(_,x){for(var b=0;b<x.length;b++){var k=x[b];k.enumerable=k.enumerable||!1,k.configurable=!0,"value"in k&&(k.writable=!0),Object.defineProperty(_,k.key,k)}}function Ee(_,x,b){return x&&$e(_.prototype,x),b&&$e(_,b),_}function qe(_,x){if(typeof x!="function"&&x!==null)throw new TypeError("Super expression must either be null or a function");_.prototype=Object.create(x&&x.prototype,{constructor:{value:_,writable:!0,configurable:!0}}),x&&or(_,x)}function or(_,x){return or=Object.setPrototypeOf||function(k,F){return k.__proto__=F,k},or(_,x)}function Bi(_){var x=Gi();return function(){var k=gt(_),F;if(x){var X=gt(this).constructor;F=Reflect.construct(k,arguments,X)}else F=k.apply(this,arguments);return Ji(this,F)}}function Ji(_,x){return x&&(j(x)==="object"||typeof x=="function")?x:Yi(_)}function Yi(_){if(_===void 0)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return _}function Gi(){if(typeof Reflect=="undefined"||!Reflect.construct||Reflect.construct.sham)return!1;if(typeof Proxy=="function")return!0;try{return Date.prototype.toString.call(Reflect.construct(Date,[],function(){})),!0}catch(_){return!1}}function gt(_){return gt=Object.setPrototypeOf?Object.getPrototypeOf:function(b){return b.__proto__||Object.getPrototypeOf(b)},gt(_)}function nr(_,x){var b="data-clipboard-".concat(_);if(!!x.hasAttribute(b))return x.getAttribute(b)}var Xi=function(_){qe(b,_);var x=Bi(b);function b(k,F){var X;return w(this,b),X=x.call(this),X.resolveOptions(F),X.listenClick(k),X}return Ee(b,[{key:"resolveOptions",value:function(){var F=arguments.length>0&&arguments[0]!==void 0?arguments[0]:{};this.action=typeof F.action=="function"?F.action:this.defaultAction,this.target=typeof F.target=="function"?F.target:this.defaultTarget,this.text=typeof F.text=="function"?F.text:this.defaultText,this.container=j(F.container)==="object"?F.container:document.body}},{key:"listenClick",value:function(F){var X=this;this.listener=l()(F,"click",function(lt){return X.onClick(lt)})}},{key:"onClick",value:function(F){var X=F.delegateTarget||F.currentTarget;this.clipboardAction&&(this.clipboardAction=null),this.clipboardAction=new K({action:this.action(X),target:this.target(X),text:this.text(X),container:this.container,trigger:X,emitter:this})}},{key:"defaultAction",value:function(F){return nr("action",F)}},{key:"defaultTarget",value:function(F){var X=nr("target",F);if(X)return document.querySelector(X)}},{key:"defaultText",value:function(F){return nr("text",F)}},{key:"destroy",value:function(){this.listener.destroy(),this.clipboardAction&&(this.clipboardAction.destroy(),this.clipboardAction=null)}}],[{key:"isSupported",value:function(){var F=arguments.length>0&&arguments[0]!==void 0?arguments[0]:["copy","cut"],X=typeof F=="string"?[F]:F,lt=!!document.queryCommandSupported;return X.forEach(function(ea){lt=lt&&!!document.queryCommandSupported(ea)}),lt}}]),b}(s()),Zi=Xi},828:function(o){var n=9;if(typeof Element!="undefined"&&!Element.prototype.matches){var i=Element.prototype;i.matches=i.matchesSelector||i.mozMatchesSelector||i.msMatchesSelector||i.oMatchesSelector||i.webkitMatchesSelector}function a(s,c){for(;s&&s.nodeType!==n;){if(typeof s.matches=="function"&&s.matches(c))return s;s=s.parentNode}}o.exports=a},438:function(o,n,i){var a=i(828);function s(p,m,f,d,v){var h=l.apply(this,arguments);return p.addEventListener(f,h,v),{destroy:function(){p.removeEventListener(f,h,v)}}}function c(p,m,f,d,v){return typeof p.addEventListener=="function"?s.apply(null,arguments):typeof f=="function"?s.bind(null,document).apply(null,arguments):(typeof p=="string"&&(p=document.querySelectorAll(p)),Array.prototype.map.call(p,function(h){return s(h,m,f,d,v)}))}function l(p,m,f,d){return function(v){v.delegateTarget=a(v.target,m),v.delegateTarget&&d.call(p,v)}}o.exports=c},879:function(o,n){n.node=function(i){return i!==void 0&&i instanceof HTMLElement&&i.nodeType===1},n.nodeList=function(i){var a=Object.prototype.toString.call(i);return i!==void 0&&(a==="[object NodeList]"||a==="[object HTMLCollection]")&&"length"in i&&(i.length===0||n.node(i[0]))},n.string=function(i){return typeof i=="string"||i instanceof String},n.fn=function(i){var a=Object.prototype.toString.call(i);return a==="[object Function]"}},370:function(o,n,i){var a=i(879),s=i(438);function c(f,d,v){if(!f&&!d&&!v)throw new Error("Missing required arguments");if(!a.string(d))throw new TypeError("Second argument must be a String");if(!a.fn(v))throw new TypeError("Third argument must be a Function");if(a.node(f))return l(f,d,v);if(a.nodeList(f))return p(f,d,v);if(a.string(f))return m(f,d,v);throw new TypeError("First argument must be a String, HTMLElement, HTMLCollection, or NodeList")}function l(f,d,v){return f.addEventListener(d,v),{destroy:function(){f.removeEventListener(d,v)}}}function p(f,d,v){return Array.prototype.forEach.call(f,function(h){h.addEventListener(d,v)}),{destroy:function(){Array.prototype.forEach.call(f,function(h){h.removeEventListener(d,v)})}}}function m(f,d,v){return s(document.body,f,d,v)}o.exports=c},817:function(o){function n(i){var a;if(i.nodeName==="SELECT")i.focus(),a=i.value;else if(i.nodeName==="INPUT"||i.nodeName==="TEXTAREA"){var s=i.hasAttribute("readonly");s||i.setAttribute("readonly",""),i.select(),i.setSelectionRange(0,i.value.length),s||i.removeAttribute("readonly"),a=i.value}else{i.hasAttribute("contenteditable")&&i.focus();var c=window.getSelection(),l=document.createRange();l.selectNodeContents(i),c.removeAllRanges(),c.addRange(l),a=c.toString()}return a}o.exports=n},279:function(o){function n(){}n.prototype={on:function(i,a,s){var c=this.e||(this.e={});return(c[i]||(c[i]=[])).push({fn:a,ctx:s}),this},once:function(i,a,s){var c=this;function l(){c.off(i,l),a.apply(s,arguments)}return l._=a,this.on(i,l,s)},emit:function(i){var a=[].slice.call(arguments,1),s=((this.e||(this.e={}))[i]||[]).slice(),c=0,l=s.length;for(c;c<l;c++)s[c].fn.apply(s[c].ctx,a);return this},off:function(i,a){var s=this.e||(this.e={}),c=s[i],l=[];if(c&&a)for(var p=0,m=c.length;p<m;p++)c[p].fn!==a&&c[p].fn._!==a&&l.push(c[p]);return l.length?s[i]=l:delete s[i],this}},o.exports=n,o.exports.TinyEmitter=n}},t={};function r(o){if(t[o])return t[o].exports;var n=t[o]={exports:{}};return e[o](n,n.exports,r),n.exports}return function(){r.n=function(o){var n=o&&o.__esModule?function(){return o.default}:function(){return o};return r.d(n,{a:n}),n}}(),function(){r.d=function(o,n){for(var i in n)r.o(n,i)&&!r.o(o,i)&&Object.defineProperty(o,i,{enumerable:!0,get:n[i]})}}(),function(){r.o=function(o,n){return Object.prototype.hasOwnProperty.call(o,n)}}(),r(134)}().default})});var Ur=wt((Gg,yi)=>{/*!
- * escape-html
- * Copyright(c) 2012-2013 TJ Holowaychuk
- * Copyright(c) 2015 Andreas Lubbe
- * Copyright(c) 2015 Tiancheng "Timothy" Gu
- * MIT Licensed
- */"use strict";var Ka=/["'&<>]/;yi.exports=Ba;function Ba(e){var t=""+e,r=Ka.exec(t);if(!r)return t;var o,n="",i=0,a=0;for(i=r.index;i<t.length;i++){switch(t.charCodeAt(i)){case 34:o="&quot;";break;case 38:o="&amp;";break;case 39:o="&#39;";break;case 60:o="&lt;";break;case 62:o="&gt;";break;default:continue}a!==i&&(n+=t.substring(a,i)),a=i+1,n+=o}return a!==i?n+t.substring(a,i):n}});var DE=Qe(Yr());var go=Qe(xo()),{__extends:Y,__assign:ps,__rest:us,__decorate:fs,__param:ms,__metadata:ds,__awaiter:yo,__generator:_t,__exportStar:hs,__createBinding:bs,__values:ce,__read:C,__spread:vs,__spreadArrays:xs,__spreadArray:H,__await:Mt,__asyncGenerator:So,__asyncDelegator:gs,__asyncValues:wo,__makeTemplateObject:ys,__importStar:Ss,__importDefault:ws,__classPrivateFieldGet:Es,__classPrivateFieldSet:Ts}=go.default;function S(e){return typeof e=="function"}function At(e){var t=function(o){Error.call(o),o.stack=new Error().stack},r=e(t);return r.prototype=Object.create(Error.prototype),r.prototype.constructor=r,r}var Lt=At(function(e){return function(r){e(this),this.message=r?r.length+` errors occurred during unsubscription:
-`+r.map(function(o,n){return n+1+") "+o.toString()}).join(`
-  `):"",this.name="UnsubscriptionError",this.errors=r}});function Te(e,t){if(e){var r=e.indexOf(t);0<=r&&e.splice(r,1)}}var ie=function(){function e(t){this.initialTeardown=t,this.closed=!1,this._parentage=null,this._teardowns=null}return e.prototype.unsubscribe=function(){var t,r,o,n,i;if(!this.closed){this.closed=!0;var a=this._parentage;if(a)if(this._parentage=null,Array.isArray(a))try{for(var s=ce(a),c=s.next();!c.done;c=s.next()){var l=c.value;l.remove(this)}}catch(h){t={error:h}}finally{try{c&&!c.done&&(r=s.return)&&r.call(s)}finally{if(t)throw t.error}}else a.remove(this);var p=this.initialTeardown;if(S(p))try{p()}catch(h){i=h instanceof Lt?h.errors:[h]}var m=this._teardowns;if(m){this._teardowns=null;try{for(var f=ce(m),d=f.next();!d.done;d=f.next()){var v=d.value;try{Eo(v)}catch(h){i=i!=null?i:[],h instanceof Lt?i=H(H([],C(i)),C(h.errors)):i.push(h)}}}catch(h){o={error:h}}finally{try{d&&!d.done&&(n=f.return)&&n.call(f)}finally{if(o)throw o.error}}}if(i)throw new Lt(i)}},e.prototype.add=function(t){var r;if(t&&t!==this)if(this.closed)Eo(t);else{if(t instanceof e){if(t.closed||t._hasParent(this))return;t._addParent(this)}(this._teardowns=(r=this._teardowns)!==null&&r!==void 0?r:[]).push(t)}},e.prototype._hasParent=function(t){var r=this._parentage;return r===t||Array.isArray(r)&&r.includes(t)},e.prototype._addParent=function(t){var r=this._parentage;this._parentage=Array.isArray(r)?(r.push(t),r):r?[r,t]:t},e.prototype._removeParent=function(t){var r=this._parentage;r===t?this._parentage=null:Array.isArray(r)&&Te(r,t)},e.prototype.remove=function(t){var r=this._teardowns;r&&Te(r,t),t instanceof e&&t._removeParent(this)},e.EMPTY=function(){var t=new e;return t.closed=!0,t}(),e}();var cr=ie.EMPTY;function kt(e){return e instanceof ie||e&&"closed"in e&&S(e.remove)&&S(e.add)&&S(e.unsubscribe)}function Eo(e){S(e)?e():e.unsubscribe()}var he={onUnhandledError:null,onStoppedNotification:null,Promise:void 0,useDeprecatedSynchronousErrorHandling:!1,useDeprecatedNextContext:!1};var Be={setTimeout:function(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=Be.delegate;return((r==null?void 0:r.setTimeout)||setTimeout).apply(void 0,H([],C(e)))},clearTimeout:function(e){var t=Be.delegate;return((t==null?void 0:t.clearTimeout)||clearTimeout)(e)},delegate:void 0};function Ct(e){Be.setTimeout(function(){var t=he.onUnhandledError;if(t)t(e);else throw e})}function Z(){}var To=function(){return lr("C",void 0,void 0)}();function Oo(e){return lr("E",void 0,e)}function _o(e){return lr("N",e,void 0)}function lr(e,t,r){return{kind:e,value:t,error:r}}var We=null;function Je(e){if(he.useDeprecatedSynchronousErrorHandling){var t=!We;if(t&&(We={errorThrown:!1,error:null}),e(),t){var r=We,o=r.errorThrown,n=r.error;if(We=null,o)throw n}}else e()}function Mo(e){he.useDeprecatedSynchronousErrorHandling&&We&&(We.errorThrown=!0,We.error=e)}var pt=function(e){Y(t,e);function t(r){var o=e.call(this)||this;return o.isStopped=!1,r?(o.destination=r,kt(r)&&r.add(o)):o.destination=sa,o}return t.create=function(r,o,n){return new ut(r,o,n)},t.prototype.next=function(r){this.isStopped?ur(_o(r),this):this._next(r)},t.prototype.error=function(r){this.isStopped?ur(Oo(r),this):(this.isStopped=!0,this._error(r))},t.prototype.complete=function(){this.isStopped?ur(To,this):(this.isStopped=!0,this._complete())},t.prototype.unsubscribe=function(){this.closed||(this.isStopped=!0,e.prototype.unsubscribe.call(this),this.destination=null)},t.prototype._next=function(r){this.destination.next(r)},t.prototype._error=function(r){try{this.destination.error(r)}finally{this.unsubscribe()}},t.prototype._complete=function(){try{this.destination.complete()}finally{this.unsubscribe()}},t}(ie);var ut=function(e){Y(t,e);function t(r,o,n){var i=e.call(this)||this,a;if(S(r))a=r;else if(r){a=r.next,o=r.error,n=r.complete;var s;i&&he.useDeprecatedNextContext?(s=Object.create(r),s.unsubscribe=function(){return i.unsubscribe()}):s=r,a=a==null?void 0:a.bind(s),o=o==null?void 0:o.bind(s),n=n==null?void 0:n.bind(s)}return i.destination={next:a?pr(a,i):Z,error:pr(o!=null?o:Ao,i),complete:n?pr(n,i):Z},i}return t}(pt);function pr(e,t){return function(){for(var r=[],o=0;o<arguments.length;o++)r[o]=arguments[o];try{e.apply(void 0,H([],C(r)))}catch(n){he.useDeprecatedSynchronousErrorHandling?Mo(n):Ct(n)}}}function Ao(e){throw e}function ur(e,t){var r=he.onStoppedNotification;r&&Be.setTimeout(function(){return r(e,t)})}var sa={closed:!0,next:Z,error:Ao,complete:Z};var Le=function(){return typeof Symbol=="function"&&Symbol.observable||"@@observable"}();function le(e){return e}function Lo(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];return fr(e)}function fr(e){return e.length===0?le:e.length===1?e[0]:function(r){return e.reduce(function(o,n){return n(o)},r)}}var A=function(){function e(t){t&&(this._subscribe=t)}return e.prototype.lift=function(t){var r=new e;return r.source=this,r.operator=t,r},e.prototype.subscribe=function(t,r,o){var n=this,i=la(t)?t:new ut(t,r,o);return Je(function(){var a=n,s=a.operator,c=a.source;i.add(s?s.call(i,c):c?n._subscribe(i):n._trySubscribe(i))}),i},e.prototype._trySubscribe=function(t){try{return this._subscribe(t)}catch(r){t.error(r)}},e.prototype.forEach=function(t,r){var o=this;return r=ko(r),new r(function(n,i){var a;a=o.subscribe(function(s){try{t(s)}catch(c){i(c),a==null||a.unsubscribe()}},i,n)})},e.prototype._subscribe=function(t){var r;return(r=this.source)===null||r===void 0?void 0:r.subscribe(t)},e.prototype[Le]=function(){return this},e.prototype.pipe=function(){for(var t=[],r=0;r<arguments.length;r++)t[r]=arguments[r];return fr(t)(this)},e.prototype.toPromise=function(t){var r=this;return t=ko(t),new t(function(o,n){var i;r.subscribe(function(a){return i=a},function(a){return n(a)},function(){return o(i)})})},e.create=function(t){return new e(t)},e}();function ko(e){var t;return(t=e!=null?e:he.Promise)!==null&&t!==void 0?t:Promise}function ca(e){return e&&S(e.next)&&S(e.error)&&S(e.complete)}function la(e){return e&&e instanceof pt||ca(e)&&kt(e)}function pa(e){return S(e==null?void 0:e.lift)}function g(e){return function(t){if(pa(t))return t.lift(function(r){try{return e(r,this)}catch(o){this.error(o)}});throw new TypeError("Unable to lift unknown Observable type")}}var y=function(e){Y(t,e);function t(r,o,n,i,a){var s=e.call(this,r)||this;return s.onFinalize=a,s._next=o?function(c){try{o(c)}catch(l){r.error(l)}}:e.prototype._next,s._error=i?function(c){try{i(c)}catch(l){r.error(l)}finally{this.unsubscribe()}}:e.prototype._error,s._complete=n?function(){try{n()}catch(c){r.error(c)}finally{this.unsubscribe()}}:e.prototype._complete,s}return t.prototype.unsubscribe=function(){var r,o=this.closed;e.prototype.unsubscribe.call(this),!o&&((r=this.onFinalize)===null||r===void 0||r.call(this))},t}(pt);var Ye={schedule:function(e){var t=requestAnimationFrame,r=cancelAnimationFrame,o=Ye.delegate;o&&(t=o.requestAnimationFrame,r=o.cancelAnimationFrame);var n=t(function(i){r=void 0,e(i)});return new ie(function(){return r==null?void 0:r(n)})},requestAnimationFrame:function(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=Ye.delegate;return((r==null?void 0:r.requestAnimationFrame)||requestAnimationFrame).apply(void 0,H([],C(e)))},cancelAnimationFrame:function(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=Ye.delegate;return((r==null?void 0:r.cancelAnimationFrame)||cancelAnimationFrame).apply(void 0,H([],C(e)))},delegate:void 0};var Co=At(function(e){return function(){e(this),this.name="ObjectUnsubscribedError",this.message="object unsubscribed"}});var T=function(e){Y(t,e);function t(){var r=e.call(this)||this;return r.closed=!1,r.observers=[],r.isStopped=!1,r.hasError=!1,r.thrownError=null,r}return t.prototype.lift=function(r){var o=new Ho(this,this);return o.operator=r,o},t.prototype._throwIfClosed=function(){if(this.closed)throw new Co},t.prototype.next=function(r){var o=this;Je(function(){var n,i;if(o._throwIfClosed(),!o.isStopped){var a=o.observers.slice();try{for(var s=ce(a),c=s.next();!c.done;c=s.next()){var l=c.value;l.next(r)}}catch(p){n={error:p}}finally{try{c&&!c.done&&(i=s.return)&&i.call(s)}finally{if(n)throw n.error}}}})},t.prototype.error=function(r){var o=this;Je(function(){if(o._throwIfClosed(),!o.isStopped){o.hasError=o.isStopped=!0,o.thrownError=r;for(var n=o.observers;n.length;)n.shift().error(r)}})},t.prototype.complete=function(){var r=this;Je(function(){if(r._throwIfClosed(),!r.isStopped){r.isStopped=!0;for(var o=r.observers;o.length;)o.shift().complete()}})},t.prototype.unsubscribe=function(){this.isStopped=this.closed=!0,this.observers=null},Object.defineProperty(t.prototype,"observed",{get:function(){var r;return((r=this.observers)===null||r===void 0?void 0:r.length)>0},enumerable:!1,configurable:!0}),t.prototype._trySubscribe=function(r){return this._throwIfClosed(),e.prototype._trySubscribe.call(this,r)},t.prototype._subscribe=function(r){return this._throwIfClosed(),this._checkFinalizedStatuses(r),this._innerSubscribe(r)},t.prototype._innerSubscribe=function(r){var o=this,n=o.hasError,i=o.isStopped,a=o.observers;return n||i?cr:(a.push(r),new ie(function(){return Te(a,r)}))},t.prototype._checkFinalizedStatuses=function(r){var o=this,n=o.hasError,i=o.thrownError,a=o.isStopped;n?r.error(i):a&&r.complete()},t.prototype.asObservable=function(){var r=new A;return r.source=this,r},t.create=function(r,o){return new Ho(r,o)},t}(A);var Ho=function(e){Y(t,e);function t(r,o){var n=e.call(this)||this;return n.destination=r,n.source=o,n}return t.prototype.next=function(r){var o,n;(n=(o=this.destination)===null||o===void 0?void 0:o.next)===null||n===void 0||n.call(o,r)},t.prototype.error=function(r){var o,n;(n=(o=this.destination)===null||o===void 0?void 0:o.error)===null||n===void 0||n.call(o,r)},t.prototype.complete=function(){var r,o;(o=(r=this.destination)===null||r===void 0?void 0:r.complete)===null||o===void 0||o.call(r)},t.prototype._subscribe=function(r){var o,n;return(n=(o=this.source)===null||o===void 0?void 0:o.subscribe(r))!==null&&n!==void 0?n:cr},t}(T);var ft={now:function(){return(ft.delegate||Date).now()},delegate:void 0};var mt=function(e){Y(t,e);function t(r,o,n){r===void 0&&(r=1/0),o===void 0&&(o=1/0),n===void 0&&(n=ft);var i=e.call(this)||this;return i._bufferSize=r,i._windowTime=o,i._timestampProvider=n,i._buffer=[],i._infiniteTimeWindow=!0,i._infiniteTimeWindow=o===1/0,i._bufferSize=Math.max(1,r),i._windowTime=Math.max(1,o),i}return t.prototype.next=function(r){var o=this,n=o.isStopped,i=o._buffer,a=o._infiniteTimeWindow,s=o._timestampProvider,c=o._windowTime;n||(i.push(r),!a&&i.push(s.now()+c)),this._trimBuffer(),e.prototype.next.call(this,r)},t.prototype._subscribe=function(r){this._throwIfClosed(),this._trimBuffer();for(var o=this._innerSubscribe(r),n=this,i=n._infiniteTimeWindow,a=n._buffer,s=a.slice(),c=0;c<s.length&&!r.closed;c+=i?1:2)r.next(s[c]);return this._checkFinalizedStatuses(r),o},t.prototype._trimBuffer=function(){var r=this,o=r._bufferSize,n=r._timestampProvider,i=r._buffer,a=r._infiniteTimeWindow,s=(a?1:2)*o;if(o<1/0&&s<i.length&&i.splice(0,i.length-s),!a){for(var c=n.now(),l=0,p=1;p<i.length&&i[p]<=c;p+=2)l=p;l&&i.splice(0,l+1)}},t}(T);var jo=function(e){Y(t,e);function t(r,o){return e.call(this)||this}return t.prototype.schedule=function(r,o){return o===void 0&&(o=0),this},t}(ie);var dt={setInterval:function(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=dt.delegate;return((r==null?void 0:r.setInterval)||setInterval).apply(void 0,H([],C(e)))},clearInterval:function(e){var t=dt.delegate;return((t==null?void 0:t.clearInterval)||clearInterval)(e)},delegate:void 0};var Ht=function(e){Y(t,e);function t(r,o){var n=e.call(this,r,o)||this;return n.scheduler=r,n.work=o,n.pending=!1,n}return t.prototype.schedule=function(r,o){if(o===void 0&&(o=0),this.closed)return this;this.state=r;var n=this.id,i=this.scheduler;return n!=null&&(this.id=this.recycleAsyncId(i,n,o)),this.pending=!0,this.delay=o,this.id=this.id||this.requestAsyncId(i,this.id,o),this},t.prototype.requestAsyncId=function(r,o,n){return n===void 0&&(n=0),dt.setInterval(r.flush.bind(r,this),n)},t.prototype.recycleAsyncId=function(r,o,n){if(n===void 0&&(n=0),n!=null&&this.delay===n&&this.pending===!1)return o;dt.clearInterval(o)},t.prototype.execute=function(r,o){if(this.closed)return new Error("executing a cancelled action");this.pending=!1;var n=this._execute(r,o);if(n)return n;this.pending===!1&&this.id!=null&&(this.id=this.recycleAsyncId(this.scheduler,this.id,null))},t.prototype._execute=function(r,o){var n=!1,i;try{this.work(r)}catch(a){n=!0,i=!!a&&a||new Error(a)}if(n)return this.unsubscribe(),i},t.prototype.unsubscribe=function(){if(!this.closed){var r=this,o=r.id,n=r.scheduler,i=n.actions;this.work=this.state=this.scheduler=null,this.pending=!1,Te(i,this),o!=null&&(this.id=this.recycleAsyncId(n,o,null)),this.delay=null,e.prototype.unsubscribe.call(this)}},t}(jo);var mr=function(){function e(t,r){r===void 0&&(r=e.now),this.schedulerActionCtor=t,this.now=r}return e.prototype.schedule=function(t,r,o){return r===void 0&&(r=0),new this.schedulerActionCtor(this,t).schedule(o,r)},e.now=ft.now,e}();var jt=function(e){Y(t,e);function t(r,o){o===void 0&&(o=mr.now);var n=e.call(this,r,o)||this;return n.actions=[],n._active=!1,n._scheduled=void 0,n}return t.prototype.flush=function(r){var o=this.actions;if(this._active){o.push(r);return}var n;this._active=!0;do if(n=r.execute(r.state,r.delay))break;while(r=o.shift());if(this._active=!1,n){for(;r=o.shift();)r.unsubscribe();throw n}},t}(mr);var ke=new jt(Ht),dr=ke;var Ro=function(e){Y(t,e);function t(r,o){var n=e.call(this,r,o)||this;return n.scheduler=r,n.work=o,n}return t.prototype.requestAsyncId=function(r,o,n){return n===void 0&&(n=0),n!==null&&n>0?e.prototype.requestAsyncId.call(this,r,o,n):(r.actions.push(this),r._scheduled||(r._scheduled=Ye.requestAnimationFrame(function(){return r.flush(void 0)})))},t.prototype.recycleAsyncId=function(r,o,n){if(n===void 0&&(n=0),n!=null&&n>0||n==null&&this.delay>0)return e.prototype.recycleAsyncId.call(this,r,o,n);r.actions.length===0&&(Ye.cancelAnimationFrame(o),r._scheduled=void 0)},t}(Ht);var Fo=function(e){Y(t,e);function t(){return e!==null&&e.apply(this,arguments)||this}return t.prototype.flush=function(r){this._active=!0,this._scheduled=void 0;var o=this.actions,n,i=-1;r=r||o.shift();var a=o.length;do if(n=r.execute(r.state,r.delay))break;while(++i<a&&(r=o.shift()));if(this._active=!1,n){for(;++i<a&&(r=o.shift());)r.unsubscribe();throw n}},t}(jt);var G=new Fo(Ro);var be=new A(function(e){return e.complete()});function Ge(e,t){return new A(function(r){var o=0;return t.schedule(function(){o===e.length?r.complete():(r.next(e[o++]),r.closed||this.schedule())})})}var Xe=function(e){return e&&typeof e.length=="number"&&typeof e!="function"};function Rt(e){return S(e==null?void 0:e.then)}function Io(e,t){return new A(function(r){var o=new ie;return o.add(t.schedule(function(){var n=e[Le]();o.add(n.subscribe({next:function(i){o.add(t.schedule(function(){return r.next(i)}))},error:function(i){o.add(t.schedule(function(){return r.error(i)}))},complete:function(){o.add(t.schedule(function(){return r.complete()}))}}))})),o})}function Po(e,t){return new A(function(r){return t.schedule(function(){return e.then(function(o){r.add(t.schedule(function(){r.next(o),r.add(t.schedule(function(){return r.complete()}))}))},function(o){r.add(t.schedule(function(){return r.error(o)}))})})})}function ua(){return typeof Symbol!="function"||!Symbol.iterator?"@@iterator":Symbol.iterator}var Ft=ua();function $o(e,t,r,o){o===void 0&&(o=0);var n=t.schedule(function(){try{r.call(this)}catch(i){e.error(i)}},o);return e.add(n),n}function Wo(e,t){return new A(function(r){var o;return r.add(t.schedule(function(){o=e[Ft](),$o(r,t,function(){var n=o.next(),i=n.value,a=n.done;a?r.complete():(r.next(i),this.schedule())})})),function(){return S(o==null?void 0:o.return)&&o.return()}})}function It(e,t){if(!e)throw new Error("Iterable cannot be null");return new A(function(r){var o=new ie;return o.add(t.schedule(function(){var n=e[Symbol.asyncIterator]();o.add(t.schedule(function(){var i=this;n.next().then(function(a){a.done?r.complete():(r.next(a.value),i.schedule())})}))})),o})}function Pt(e){return S(e[Le])}function $t(e){return S(e==null?void 0:e[Ft])}function Wt(e){return Symbol.asyncIterator&&S(e==null?void 0:e[Symbol.asyncIterator])}function Ut(e){return new TypeError("You provided "+(e!==null&&typeof e=="object"?"an invalid object":"'"+e+"'")+" where a stream was expected. You can provide an Observable, Promise, ReadableStream, Array, AsyncIterable, or Iterable.")}function Vt(e){return So(this,arguments,function(){var r,o,n,i;return _t(this,function(a){switch(a.label){case 0:r=e.getReader(),a.label=1;case 1:a.trys.push([1,,9,10]),a.label=2;case 2:return[4,Mt(r.read())];case 3:return o=a.sent(),n=o.value,i=o.done,i?[4,Mt(void 0)]:[3,5];case 4:return[2,a.sent()];case 5:return[4,Mt(n)];case 6:return[4,a.sent()];case 7:return a.sent(),[3,2];case 8:return[3,10];case 9:return r.releaseLock(),[7];case 10:return[2]}})})}function Nt(e){return S(e==null?void 0:e.getReader)}function Uo(e,t){return It(Vt(e),t)}function Vo(e,t){if(e!=null){if(Pt(e))return Io(e,t);if(Xe(e))return Ge(e,t);if(Rt(e))return Po(e,t);if(Wt(e))return It(e,t);if($t(e))return Wo(e,t);if(Nt(e))return Uo(e,t)}throw Ut(e)}function ge(e,t){return t?Vo(e,t):D(e)}function D(e){if(e instanceof A)return e;if(e!=null){if(Pt(e))return fa(e);if(Xe(e))return hr(e);if(Rt(e))return ma(e);if(Wt(e))return No(e);if($t(e))return da(e);if(Nt(e))return ha(e)}throw Ut(e)}function fa(e){return new A(function(t){var r=e[Le]();if(S(r.subscribe))return r.subscribe(t);throw new TypeError("Provided object does not correctly implement Symbol.observable")})}function hr(e){return new A(function(t){for(var r=0;r<e.length&&!t.closed;r++)t.next(e[r]);t.complete()})}function ma(e){return new A(function(t){e.then(function(r){t.closed||(t.next(r),t.complete())},function(r){return t.error(r)}).then(null,Ct)})}function da(e){return new A(function(t){var r,o;try{for(var n=ce(e),i=n.next();!i.done;i=n.next()){var a=i.value;if(t.next(a),t.closed)return}}catch(s){r={error:s}}finally{try{i&&!i.done&&(o=n.return)&&o.call(n)}finally{if(r)throw r.error}}t.complete()})}function No(e){return new A(function(t){ba(e,t).catch(function(r){return t.error(r)})})}function ha(e){return No(Vt(e))}function ba(e,t){var r,o,n,i;return yo(this,void 0,void 0,function(){var a,s;return _t(this,function(c){switch(c.label){case 0:c.trys.push([0,5,6,11]),r=wo(e),c.label=1;case 1:return[4,r.next()];case 2:if(o=c.sent(),!!o.done)return[3,4];if(a=o.value,t.next(a),t.closed)return[2];c.label=3;case 3:return[3,1];case 4:return[3,11];case 5:return s=c.sent(),n={error:s},[3,11];case 6:return c.trys.push([6,,9,10]),o&&!o.done&&(i=r.return)?[4,i.call(r)]:[3,8];case 7:c.sent(),c.label=8;case 8:return[3,10];case 9:if(n)throw n.error;return[7];case 10:return[7];case 11:return t.complete(),[2]}})})}function ye(e,t){return t?Ge(e,t):hr(e)}function Dt(e){return e&&S(e.schedule)}function br(e){return e[e.length-1]}function Ce(e){return S(br(e))?e.pop():void 0}function fe(e){return Dt(br(e))?e.pop():void 0}function zt(e,t){return typeof br(e)=="number"?e.pop():t}function R(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=fe(e);return r?Ge(e,r):ye(e)}function Do(e){return e instanceof Date&&!isNaN(e)}function u(e,t){return g(function(r,o){var n=0;r.subscribe(new y(o,function(i){o.next(e.call(t,i,n++))}))})}var va=Array.isArray;function xa(e,t){return va(t)?e.apply(void 0,H([],C(t))):e(t)}function He(e){return u(function(t){return xa(e,t)})}function z(e,t){return t===void 0&&(t=0),g(function(r,o){r.subscribe(new y(o,function(n){return o.add(e.schedule(function(){return o.next(n)},t))},function(){return o.add(e.schedule(function(){return o.complete()},t))},function(n){return o.add(e.schedule(function(){return o.error(n)},t))}))})}var ga=Array.isArray,ya=Object.getPrototypeOf,Sa=Object.prototype,wa=Object.keys;function zo(e){if(e.length===1){var t=e[0];if(ga(t))return{args:t,keys:null};if(Ea(t)){var r=wa(t);return{args:r.map(function(o){return t[o]}),keys:r}}}return{args:e,keys:null}}function Ea(e){return e&&typeof e=="object"&&ya(e)===Sa}function qo(e,t){return e.reduce(function(r,o,n){return r[o]=t[n],r},{})}function q(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=fe(e),o=Ce(e),n=zo(e),i=n.args,a=n.keys;if(i.length===0)return ge([],r);var s=new A(vr(i,r,a?function(c){return qo(a,c)}:le));return o?s.pipe(He(o)):s}function vr(e,t,r){return r===void 0&&(r=le),function(o){Qo(t,function(){for(var n=e.length,i=new Array(n),a=n,s=n,c=function(p){Qo(t,function(){var m=ge(e[p],t),f=!1;m.subscribe(new y(o,function(d){i[p]=d,f||(f=!0,s--),s||o.next(r(i.slice()))},function(){--a||o.complete()}))},o)},l=0;l<n;l++)c(l)},o)}}function Qo(e,t,r){e?r.add(e.schedule(t)):t()}function Ko(e,t,r,o,n,i,a,s){var c=[],l=0,p=0,m=!1,f=function(){m&&!c.length&&!l&&t.complete()},d=function(h){return l<o?v(h):c.push(h)},v=function(h){i&&t.next(h),l++;var $=!1;D(r(h,p++)).subscribe(new y(t,function(K){n==null||n(K),i?d(K):t.next(K)},function(){$=!0},void 0,function(){if($)try{l--;for(var K=function(){var j=c.shift();a?t.add(a.schedule(function(){return v(j)})):v(j)};c.length&&l<o;)K();f()}catch(j){t.error(j)}}))};return e.subscribe(new y(t,d,function(){m=!0,f()})),function(){s==null||s()}}function ee(e,t,r){return r===void 0&&(r=1/0),S(t)?ee(function(o,n){return u(function(i,a){return t(o,i,n,a)})(D(e(o,n)))},r):(typeof t=="number"&&(r=t),g(function(o,n){return Ko(o,n,e,r)}))}function Ze(e){return e===void 0&&(e=1/0),ee(le,e)}function Bo(){return Ze(1)}function ht(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];return Bo()(ye(e,fe(e)))}function Oe(e){return new A(function(t){D(e()).subscribe(t)})}var Ta=["addListener","removeListener"],Oa=["addEventListener","removeEventListener"],_a=["on","off"];function E(e,t,r,o){if(S(r)&&(o=r,r=void 0),o)return E(e,t,r).pipe(He(o));var n=C(La(e)?Oa.map(function(s){return function(c){return e[s](t,c,r)}}):Ma(e)?Ta.map(Jo(e,t)):Aa(e)?_a.map(Jo(e,t)):[],2),i=n[0],a=n[1];if(!i&&Xe(e))return ee(function(s){return E(s,t,r)})(ye(e));if(!i)throw new TypeError("Invalid event target");return new A(function(s){var c=function(){for(var l=[],p=0;p<arguments.length;p++)l[p]=arguments[p];return s.next(1<l.length?l:l[0])};return i(c),function(){return a(c)}})}function Jo(e,t){return function(r){return function(o){return e[r](t,o)}}}function Ma(e){return S(e.addListener)&&S(e.removeListener)}function Aa(e){return S(e.on)&&S(e.off)}function La(e){return S(e.addEventListener)&&S(e.removeEventListener)}function qt(e,t,r){return r?qt(e,t).pipe(He(r)):new A(function(o){var n=function(){for(var a=[],s=0;s<arguments.length;s++)a[s]=arguments[s];return o.next(a.length===1?a[0]:a)},i=e(n);return S(t)?function(){return t(n,i)}:void 0})}function Yo(e,t,r){e===void 0&&(e=0),r===void 0&&(r=dr);var o=-1;return t!=null&&(Dt(t)?r=t:o=t),new A(function(n){var i=Do(e)?+e-r.now():e;i<0&&(i=0);var a=0;return r.schedule(function(){n.closed||(n.next(a++),0<=o?this.schedule(void 0,o):n.complete())},i)})}function P(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=fe(e),o=zt(e,1/0),n=e;return n.length?n.length===1?D(n[0]):Ze(o)(ye(n,r)):be}var B=new A(Z);var ka=Array.isArray;function et(e){return e.length===1&&ka(e[0])?e[0]:e}function M(e,t){return g(function(r,o){var n=0;r.subscribe(new y(o,function(i){return e.call(t,i,n++)&&o.next(i)}))})}function bt(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=Ce(e),o=et(e);return o.length?new A(function(n){var i=o.map(function(){return[]}),a=o.map(function(){return!1});n.add(function(){i=a=null});for(var s=function(l){D(o[l]).subscribe(new y(n,function(p){if(i[l].push(p),i.every(function(f){return f.length})){var m=i.map(function(f){return f.shift()});n.next(r?r.apply(void 0,H([],C(m))):m),i.some(function(f,d){return!f.length&&a[d]})&&n.complete()}},function(){a[l]=!0,!i[l].length&&n.complete()}))},c=0;!n.closed&&c<o.length;c++)s(c);return function(){i=a=null}}):be}function ve(e,t){return t===void 0&&(t=null),t=t!=null?t:e,g(function(r,o){var n=[],i=0;r.subscribe(new y(o,function(a){var s,c,l,p,m=null;i++%t==0&&n.push([]);try{for(var f=ce(n),d=f.next();!d.done;d=f.next()){var v=d.value;v.push(a),e<=v.length&&(m=m!=null?m:[],m.push(v))}}catch(K){s={error:K}}finally{try{d&&!d.done&&(c=f.return)&&c.call(f)}finally{if(s)throw s.error}}if(m)try{for(var h=ce(m),$=h.next();!$.done;$=h.next()){var v=$.value;Te(n,v),o.next(v)}}catch(K){l={error:K}}finally{try{$&&!$.done&&(p=h.return)&&p.call(h)}finally{if(l)throw l.error}}},function(){var a,s;try{for(var c=ce(n),l=c.next();!l.done;l=c.next()){var p=l.value;o.next(p)}}catch(m){a={error:m}}finally{try{l&&!l.done&&(s=c.return)&&s.call(c)}finally{if(a)throw a.error}}o.complete()},void 0,function(){n=null}))})}function tt(e){return g(function(t,r){var o=null,n=!1,i;o=t.subscribe(new y(r,void 0,void 0,function(a){i=D(e(a,tt(e)(t))),o?(o.unsubscribe(),o=null,i.subscribe(r)):n=!0})),n&&(o.unsubscribe(),o=null,i.subscribe(r))})}function Go(e,t,r,o,n){return function(i,a){var s=r,c=t,l=0;i.subscribe(new y(a,function(p){var m=l++;c=s?e(c,p,m):(s=!0,p),o&&a.next(c)},n&&function(){s&&a.next(c),a.complete()}))}}function xr(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=Ce(e);return r?Lo(xr.apply(void 0,H([],C(e))),He(r)):g(function(o,n){vr(H([o],C(et(e))))(n)})}function rt(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];return xr.apply(void 0,H([],C(e)))}function gr(e,t){return S(t)?ee(e,t,1):ee(e,1)}function yr(e,t){return t===void 0&&(t=ke),g(function(r,o){var n=null,i=null,a=null,s=function(){if(n){n.unsubscribe(),n=null;var l=i;i=null,o.next(l)}};function c(){var l=a+e,p=t.now();if(p<l){n=this.schedule(void 0,l-p),o.add(n);return}s()}r.subscribe(new y(o,function(l){i=l,a=t.now(),n||(n=t.schedule(c,e),o.add(n))},function(){s(),o.complete()},void 0,function(){i=n=null}))})}function Ue(e){return g(function(t,r){var o=!1;t.subscribe(new y(r,function(n){o=!0,r.next(n)},function(){o||r.next(e),r.complete()}))})}function me(e){return e<=0?function(){return be}:g(function(t,r){var o=0;t.subscribe(new y(r,function(n){++o<=e&&(r.next(n),e<=o&&r.complete())}))})}function Xo(){return g(function(e,t){e.subscribe(new y(t,Z))})}function re(e){return u(function(){return e})}function Sr(e,t){return t?function(r){return ht(t.pipe(me(1),Xo()),r.pipe(Sr(e)))}:ee(function(r,o){return e(r,o).pipe(me(1),re(r))})}function _e(e,t){t===void 0&&(t=ke);var r=Yo(e,t);return Sr(function(){return r})}function N(e,t){return t===void 0&&(t=le),e=e!=null?e:Ca,g(function(r,o){var n,i=!0;r.subscribe(new y(o,function(a){var s=t(a);(i||!e(n,s))&&(i=!1,n=s,o.next(a))}))})}function Ca(e,t){return e===t}function U(e,t){return N(function(r,o){return t?t(r[e],o[e]):r[e]===o[e]})}function Qt(e){return g(function(t,r){try{t.subscribe(r)}finally{r.add(e)}})}function wr(e){return e<=0?function(){return be}:g(function(t,r){var o=[];t.subscribe(new y(r,function(n){o.push(n),e<o.length&&o.shift()},function(){var n,i;try{for(var a=ce(o),s=a.next();!s.done;s=a.next()){var c=s.value;r.next(c)}}catch(l){n={error:l}}finally{try{s&&!s.done&&(i=a.return)&&i.call(a)}finally{if(n)throw n.error}}r.complete()},void 0,function(){o=null}))})}function Zo(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=fe(e),o=zt(e,1/0);return e=et(e),g(function(n,i){Ze(o)(ye(H([n],C(e)),r)).subscribe(i)})}function Ve(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];return Zo.apply(void 0,H([],C(e)))}function ot(e){return g(function(t,r){var o=!1,n=null;t.subscribe(new y(r,function(a){o=!0,n=a}));var i=function(){if(o){o=!1;var a=n;n=null,r.next(a)}};e.subscribe(new y(r,i,Z))})}function Er(e,t){return g(Go(e,t,arguments.length>=2,!0))}function ne(e){e===void 0&&(e={});var t=e.connector,r=t===void 0?function(){return new T}:t,o=e.resetOnError,n=o===void 0?!0:o,i=e.resetOnComplete,a=i===void 0?!0:i,s=e.resetOnRefCountZero,c=s===void 0?!0:s;return function(l){var p=null,m=null,f=null,d=0,v=!1,h=!1,$=function(){m==null||m.unsubscribe(),m=null},K=function(){$(),p=f=null,v=h=!1},j=function(){var w=p;K(),w==null||w.unsubscribe()};return g(function(w,$e){d++,!h&&!v&&$();var Ee=f=f!=null?f:r();$e.add(function(){d--,d===0&&!h&&!v&&(m=Tr(j,c))}),Ee.subscribe($e),p||(p=new ut({next:function(qe){return Ee.next(qe)},error:function(qe){h=!0,$(),m=Tr(K,n,qe),Ee.error(qe)},complete:function(){v=!0,$(),m=Tr(K,a),Ee.complete()}}),ge(w).subscribe(p))})(l)}}function Tr(e,t){for(var r=[],o=2;o<arguments.length;o++)r[o-2]=arguments[o];return t===!0?(e(),null):t===!1?null:t.apply(void 0,H([],C(r))).pipe(me(1)).subscribe(function(){return e()})}function te(e,t,r){var o,n,i,a=!1;return e&&typeof e=="object"?(i=(o=e.bufferSize)!==null&&o!==void 0?o:1/0,t=(n=e.windowTime)!==null&&n!==void 0?n:1/0,a=!!e.refCount,r=e.scheduler):i=e!=null?e:1/0,ne({connector:function(){return new mt(i,t,r)},resetOnError:!0,resetOnComplete:!1,resetOnRefCountZero:a})}function Kt(e){return M(function(t,r){return e<=r})}function Or(e){return g(function(t,r){var o=!1,n=new y(r,function(){n==null||n.unsubscribe(),o=!0},Z);D(e).subscribe(n),t.subscribe(new y(r,function(i){return o&&r.next(i)}))})}function V(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=fe(e);return g(function(o,n){(r?ht(e,o,r):ht(e,o)).subscribe(n)})}function O(e,t){return g(function(r,o){var n=null,i=0,a=!1,s=function(){return a&&!n&&o.complete()};r.subscribe(new y(o,function(c){n==null||n.unsubscribe();var l=0,p=i++;D(e(c,p)).subscribe(n=new y(o,function(m){return o.next(t?t(c,m,p,l++):m)},function(){n=null,s()}))},function(){a=!0,s()}))})}function _r(e,t){return S(t)?O(function(){return e},t):O(function(){return e})}function Mr(e){return g(function(t,r){D(e).subscribe(new y(r,function(){return r.complete()},Z)),!r.closed&&t.subscribe(r)})}function Ar(e,t){return t===void 0&&(t=!1),g(function(r,o){var n=0;r.subscribe(new y(o,function(i){var a=e(i,n++);(a||t)&&o.next(i),!a&&o.complete()}))})}function L(e,t,r){var o=S(e)||t||r?{next:e,error:t,complete:r}:e;return o?g(function(n,i){var a;(a=o.subscribe)===null||a===void 0||a.call(o);var s=!0;n.subscribe(new y(i,function(c){var l;(l=o.next)===null||l===void 0||l.call(o,c),i.next(c)},function(){var c;s=!1,(c=o.complete)===null||c===void 0||c.call(o),i.complete()},function(c){var l;s=!1,(l=o.error)===null||l===void 0||l.call(o,c),i.error(c)},function(){var c,l;s&&((c=o.unsubscribe)===null||c===void 0||c.call(o)),(l=o.finalize)===null||l===void 0||l.call(o)}))}):le}var Ha={leading:!0,trailing:!1};function Lr(e,t){var r=t===void 0?Ha:t,o=r.leading,n=r.trailing;return g(function(i,a){var s=!1,c=null,l=null,p=!1,m=function(){l==null||l.unsubscribe(),l=null,n&&(v(),p&&a.complete())},f=function(){l=null,p&&a.complete()},d=function(h){return l=D(e(h)).subscribe(new y(a,m,f))},v=function(){if(s){s=!1;var h=c;c=null,a.next(h),!p&&d(h)}};i.subscribe(new y(a,function(h){s=!0,c=h,!(l&&!l.closed)&&(o?v():d(h))},function(){p=!0,!(n&&s&&l&&!l.closed)&&a.complete()}))})}function pe(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];var r=Ce(e);return g(function(o,n){for(var i=e.length,a=new Array(i),s=e.map(function(){return!1}),c=!1,l=function(m){D(e[m]).subscribe(new y(n,function(f){a[m]=f,!c&&!s[m]&&(s[m]=!0,(c=s.every(le))&&(s=null))},Z))},p=0;p<i;p++)l(p);o.subscribe(new y(n,function(m){if(c){var f=H([m],C(a));n.next(r?r.apply(void 0,H([],C(f))):f)}}))})}function en(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];return g(function(r,o){bt.apply(void 0,H([r],C(e))).subscribe(o)})}function kr(){for(var e=[],t=0;t<arguments.length;t++)e[t]=arguments[t];return en.apply(void 0,H([],C(e)))}function tn(){let e=new mt;return E(document,"DOMContentLoaded").pipe(re(document)).subscribe(e),e}function ae(e,t=document){return t.querySelector(e)||void 0}function de(e,t=document){let r=ae(e,t);if(typeof r=="undefined")throw new ReferenceError(`Missing element: expected "${e}" to be present`);return r}function Ne(){return document.activeElement instanceof HTMLElement?document.activeElement:void 0}function Q(e,t=document){return Array.from(t.querySelectorAll(e))}function nt(e){return document.createElement(e)}function De(e,...t){e.replaceWith(...t)}function xe(e,t=!0){t?e.focus():e.blur()}function rn(e){return P(E(e,"focus"),E(e,"blur")).pipe(u(({type:t})=>t==="focus"),V(e===Ne()))}var on=new T,ja=Oe(()=>R(new ResizeObserver(e=>{for(let t of e)on.next(t)}))).pipe(O(e=>B.pipe(V(e)).pipe(Qt(()=>e.disconnect()))),te(1));function je(e){return{width:e.offsetWidth,height:e.offsetHeight}}function Bt(e){return{width:e.scrollWidth,height:e.scrollHeight}}function Re(e){return ja.pipe(L(t=>t.observe(e)),O(t=>on.pipe(M(({target:r})=>r===e),Qt(()=>t.unobserve(e)),u(()=>je(e)))),V(je(e)))}function nn(e){return{x:e.scrollLeft,y:e.scrollTop}}function Ra(e){return P(E(e,"scroll"),E(window,"resize")).pipe(u(()=>nn(e)),V(nn(e)))}function an(e,t=16){return Ra(e).pipe(u(({y:r})=>{let o=je(e),n=Bt(e);return r>=n.height-o.height-t}),N())}function sn(e){if(e instanceof HTMLInputElement)e.select();else throw new Error("Not implemented")}var Jt={drawer:de("[data-md-toggle=drawer]"),search:de("[data-md-toggle=search]")};function cn(e){return Jt[e].checked}function Fe(e,t){Jt[e].checked!==t&&Jt[e].click()}function Yt(e){let t=Jt[e];return E(t,"change").pipe(u(()=>t.checked),V(t.checked))}function Fa(e){switch(e.tagName){case"INPUT":case"SELECT":case"TEXTAREA":return!0;default:return e.isContentEditable}}function ln(){return E(window,"keydown").pipe(M(e=>!(e.metaKey||e.ctrlKey)),u(e=>({mode:cn("search")?"search":"global",type:e.key,claim(){e.preventDefault(),e.stopPropagation()}})),M(({mode:e})=>{if(e==="global"){let t=Ne();if(typeof t!="undefined")return!Fa(t)}return!0}),ne())}function Ie(){return new URL(location.href)}function pn(e){location.href=e.href}function un(){return new T}function fn(){return location.hash.substring(1)}function mn(e){let t=nt("a");t.href=e,t.addEventListener("click",r=>r.stopPropagation()),t.click()}function Ia(){return E(window,"hashchange").pipe(u(fn),V(fn()),M(e=>e.length>0),ne())}function dn(){return Ia().pipe(O(e=>R(ae(`[id="${e}"]`))))}function vt(e){let t=matchMedia(e);return qt(r=>t.addListener(()=>r(t.matches))).pipe(V(t.matches))}function hn(){return E(window,"beforeprint").pipe(re(void 0))}function Cr(e,t){return e.pipe(O(r=>r?t():B))}function Gt(e,t={credentials:"same-origin"}){return ge(fetch(`${e}`,t)).pipe(M(r=>r.status===200))}function Se(e,t){return Gt(e,t).pipe(O(r=>r.json()),te(1))}function bn(e,t){let r=new DOMParser;return Gt(e,t).pipe(O(o=>o.text()),u(o=>r.parseFromString(o,"text/xml")),te(1))}function vn(){return{x:Math.max(0,pageXOffset),y:Math.max(0,pageYOffset)}}function Hr({x:e,y:t}){window.scrollTo(e||0,t||0)}function xn(){return P(E(window,"scroll",{passive:!0}),E(window,"resize",{passive:!0})).pipe(u(vn),V(vn()))}function gn(){return{width:innerWidth,height:innerHeight}}function yn(){return E(window,"resize",{passive:!0}).pipe(u(gn),V(gn()))}function Sn(){return q([xn(),yn()]).pipe(u(([e,t])=>({offset:e,size:t})),te(1))}function Xt(e,{viewport$:t,header$:r}){let o=t.pipe(U("size")),n=q([o,r]).pipe(u(()=>({x:e.offsetLeft,y:e.offsetTop})));return q([r,t,n]).pipe(u(([{height:i},{offset:a,size:s},{x:c,y:l}])=>({offset:{x:a.x-c,y:a.y-l+i},size:s})))}function wn(e,{tx$:t}){let r=E(e,"message").pipe(u(({data:o})=>o));return t.pipe(Lr(()=>r,{leading:!0,trailing:!0}),L(o=>e.postMessage(o)),_r(r),ne())}var Pa=de("#__config"),it=JSON.parse(Pa.textContent);it.base=`${new URL(it.base,Ie())}`;function se(){return it}function Me(e){return it.features.includes(e)}function J(e,t){return typeof t!="undefined"?it.translations[e].replace("#",t.toString()):it.translations[e]}function we(e,t=document){return de(`[data-md-component=${e}]`,t)}function oe(e,t=document){return Q(`[data-md-component=${e}]`,t)}var ai=Qe(Rr());function En(e,t=0){e.setAttribute("tabindex",t.toString())}function Tn(e){e.removeAttribute("tabindex")}function On(e,t){e.setAttribute("data-md-state","lock"),e.style.top=`-${t}px`}function _n(e){let t=-1*parseInt(e.style.top,10);e.removeAttribute("data-md-state"),e.style.top="",t&&window.scrollTo(0,t)}function Mn(e,t){e.setAttribute("data-md-state",t)}function An(e){e.removeAttribute("data-md-state")}function Ln(e,t){e.classList.toggle("md-nav__link--active",t)}function kn(e){e.classList.remove("md-nav__link--active")}function Cn(e,t){e.firstElementChild.innerHTML=t}function Hn(e,t){e.setAttribute("data-md-state",t)}function jn(e){e.removeAttribute("data-md-state")}function Rn(e,t){e.setAttribute("data-md-state",t)}function Fn(e){e.removeAttribute("data-md-state")}function In(e,t){e.setAttribute("data-md-state",t)}function Pn(e){e.removeAttribute("data-md-state")}function $n(e,t){e.placeholder=t}function Wn(e){e.placeholder=J("search.placeholder")}function Un(e,t){if(typeof t=="string"||typeof t=="number")e.innerHTML+=t.toString();else if(t instanceof Node)e.appendChild(t);else if(Array.isArray(t))for(let r of t)Un(e,r)}function W(e,t,...r){let o=document.createElement(e);if(t)for(let n of Object.keys(t))typeof t[n]!="boolean"?o.setAttribute(n,t[n]):t[n]&&o.setAttribute(n,"");for(let n of r)Un(o,n);return o}function Vn(e,t){let r=t;if(e.length>r){for(;e[r]!==" "&&--r>0;);return`${e.substring(0,r)}...`}return e}function Zt(e){if(e>999){let t=+((e-950)%1e3>99);return`${((e+1e-6)/1e3).toFixed(t)}k`}else return e.toString()}function Nn(e,t){switch(t){case 0:e.textContent=J("search.result.none");break;case 1:e.textContent=J("search.result.one");break;default:e.textContent=J("search.result.other",Zt(t))}}function Fr(e){e.textContent=J("search.result.placeholder")}function Dn(e,t){e.appendChild(t)}function zn(e){e.innerHTML=""}function qn(e,t){e.style.top=`${t}px`}function Qn(e){e.style.top=""}function Kn(e,t){let r=e.firstElementChild;r.style.height=`${t-2*r.offsetTop}px`}function Bn(e){let t=e.firstElementChild;t.style.height=""}function Jn(e,t){e.lastElementChild.appendChild(t)}function Yn(e,t){e.lastElementChild.setAttribute("data-md-state",t)}function Gn(e,t){e.setAttribute("data-md-state",t)}function Ir(e){e.removeAttribute("data-md-state")}function Xn(e,t){e.setAttribute("data-md-state",t)}function Pr(e){e.removeAttribute("data-md-state")}function Zn(e,t){e.style.top=`${t}px`}function ei(e){e.style.top=""}function ti(e){return W("button",{class:"md-clipboard md-icon",title:J("clipboard.copy"),"data-clipboard-target":`#${e} > code`})}var ze;(function(r){r[r.TEASER=1]="TEASER",r[r.PARENT=2]="PARENT"})(ze||(ze={}));function $r(e,t){let r=t&2,o=t&1,n=Object.keys(e.terms).filter(a=>!e.terms[a]).map(a=>[W("del",null,a)," "]).flat().slice(0,-1),i=new URL(e.location);return Me("search.highlight")&&i.searchParams.set("h",Object.entries(e.terms).filter(([,a])=>a).reduce((a,[s])=>`${a} ${s}`.trim(),"")),W("a",{href:`${i}`,class:"md-search-result__link",tabIndex:-1},W("article",{class:["md-search-result__article",...r?["md-search-result__article--document"]:[]].join(" "),"data-md-score":e.score.toFixed(2)},r>0&&W("div",{class:"md-search-result__icon md-icon"}),W("h1",{class:"md-search-result__title"},e.title),o>0&&e.text.length>0&&W("p",{class:"md-search-result__teaser"},Vn(e.text,320)),o>0&&n.length>0&&W("p",{class:"md-search-result__terms"},J("search.result.term.missing"),": ",n)))}function ri(e){let t=e[0].score,r=[...e],o=r.findIndex(l=>!l.location.includes("#")),[n]=r.splice(o,1),i=r.findIndex(l=>l.score<t);i===-1&&(i=r.length);let a=r.slice(0,i),s=r.slice(i),c=[$r(n,2|+(!o&&i===0)),...a.map(l=>$r(l,1)),...s.length?[W("details",{class:"md-search-result__more"},W("summary",{tabIndex:-1},s.length>0&&s.length===1?J("search.result.more.one"):J("search.result.more.other",s.length)),s.map(l=>$r(l,1)))]:[]];return W("li",{class:"md-search-result__item"},c)}function oi(e){return W("ul",{class:"md-source__facts"},Object.entries(e).map(([t,r])=>W("li",{class:`md-source__fact md-source__fact--${t}`},typeof r=="number"?Zt(r):r)))}function ni(e){return W("div",{class:"md-typeset__scrollwrap"},W("div",{class:"md-typeset__table"},e))}function $a(e){let t=se(),r=new URL(`../${e.version}/`,t.base);return W("li",{class:"md-version__item"},W("a",{href:r.toString(),class:"md-version__link"},e.title))}function ii(e){let t=se(),[,r]=t.base.match(/([^/]+)\/?$/),o=e.find(({version:n,aliases:i})=>n===r||i.includes(r))||e[0];return W("div",{class:"md-version"},W("button",{class:"md-version__current","aria-label":J("select.version.title")},o.title),W("ul",{class:"md-version__list"},e.map($a)))}var Wa=0;function Ua(e,{viewport$:t}){let r=R(e).pipe(O(o=>{let n=o.closest("[data-tabs]");return n instanceof HTMLElement?P(...Q("input",n).map(i=>E(i,"change"))):B}));return P(t.pipe(U("size")),r).pipe(u(()=>{let o=je(e);return{scroll:Bt(e).width>o.width}}),U("scroll"))}function si(e,t){let r=new T;if(r.pipe(pe(vt("(hover)"))).subscribe(([{scroll:o},n])=>{o&&n?En(e):Tn(e)}),ai.default.isSupported()){let o=e.closest("pre");o.id=`__code_${Wa++}`,o.insertBefore(ti(o.id),e)}return Ua(e,t).pipe(L(r),u(o=>I({ref:e},o)))}function Va(e,{target$:t,print$:r}){return t.pipe(u(o=>o.closest("details:not([open])")),M(o=>e===o),Ve(r),re(e))}function ci(e,t){let r=new T;return r.subscribe(()=>{e.setAttribute("open",""),e.scrollIntoView()}),Va(e,t).pipe(L(r),re({ref:e}))}var li=nt("table");function pi(e){return De(e,li),De(li,ni(e)),R({ref:e})}function ui(e,{target$:t,viewport$:r,print$:o}){return P(...Q("pre > code",e).map(n=>si(n,{viewport$:r})),...Q("table:not([class])",e).map(n=>pi(n)),...Q("details",e).map(n=>ci(n,{target$:t,print$:o})))}function Na(e,{alert$:t}){return t.pipe(O(r=>P(R(!0),R(!1).pipe(_e(2e3))).pipe(u(o=>({message:r,open:o})))))}function fi(e,t){let r=new T;return r.pipe(z(G)).subscribe(({message:o,open:n})=>{Cn(e,o),n?Hn(e,"open"):jn(e)}),Na(e,t).pipe(L(r),u(o=>I({ref:e},o)))}function Da({viewport$:e}){if(!Me("header.autohide"))return R(!1);let t=e.pipe(u(({offset:{y:n}})=>n),ve(2,1),u(([n,i])=>[n<i,i]),U(0)),r=q([e,t]).pipe(M(([{offset:n},[,i]])=>Math.abs(i-n.y)>100),u(([,[n]])=>n),N()),o=Yt("search");return q([e,o]).pipe(u(([{offset:n},i])=>n.y>400&&!i),N(),O(n=>n?r:R(!1)),V(!1))}function mi(e,t){return Oe(()=>{let r=getComputedStyle(e);return R(r.position==="sticky"||r.position==="-webkit-sticky")}).pipe(rt(Re(e),Da(t)),u(([r,{height:o},n])=>({height:r?o:0,sticky:r,hidden:n})),N((r,o)=>r.sticky===o.sticky&&r.height===o.height&&r.hidden===o.hidden),te(1))}function di(e,{header$:t,main$:r}){let o=new T;return o.pipe(U("active"),rt(t),z(G)).subscribe(([{active:n},{hidden:i}])=>{n?Rn(e,i?"hidden":"shadow"):Fn(e)}),r.subscribe(n=>o.next(n)),t.pipe(u(n=>I({ref:e},n)))}function za(e,{viewport$:t,header$:r}){return Xt(e,{header$:r,viewport$:t}).pipe(u(({offset:{y:o}})=>{let{height:n}=je(e);return{active:o>=n}}),U("active"))}function hi(e,t){let r=new T;r.pipe(z(G)).subscribe(({active:n})=>{n?In(e,"active"):Pn(e)});let o=ae("article h1");return typeof o=="undefined"?B:za(o,t).pipe(L(r),u(n=>I({ref:e},n)))}function bi(e,{viewport$:t,header$:r}){let o=r.pipe(u(({height:i})=>i),N()),n=o.pipe(O(()=>Re(e).pipe(u(({height:i})=>({top:e.offsetTop,bottom:e.offsetTop+i})),U("bottom"))));return q([o,n,t]).pipe(u(([i,{top:a,bottom:s},{offset:{y:c},size:{height:l}}])=>(l=Math.max(0,l-Math.max(0,a-c,i)-Math.max(0,l+c-s)),{offset:a-i,height:l,active:a-i<=c})),N((i,a)=>i.offset===a.offset&&i.height===a.height&&i.active===a.active))}function qa(e){let t=localStorage.getItem(__prefix("__palette")),r=JSON.parse(t)||{index:e.findIndex(n=>matchMedia(n.getAttribute("data-md-color-media")).matches)},o=R(...e).pipe(ee(n=>E(n,"change").pipe(re(n))),V(e[Math.max(0,r.index)]),u(n=>({index:e.indexOf(n),color:{scheme:n.getAttribute("data-md-color-scheme"),primary:n.getAttribute("data-md-color-primary"),accent:n.getAttribute("data-md-color-accent")}})),te(1));return o.subscribe(n=>{localStorage.setItem(__prefix("__palette"),JSON.stringify(n))}),o}function vi(e){let t=new T;t.subscribe(o=>{for(let[n,i]of Object.entries(o.color))typeof i=="string"&&document.body.setAttribute(`data-md-color-${n}`,i);for(let n=0;n<r.length;n++){let i=r[n].nextElementSibling;i instanceof HTMLElement&&(i.hidden=o.index!==n)}});let r=Q("input",e);return qa(r).pipe(L(t),u(o=>I({ref:e},o)))}var Wr=Qe(Rr());function xi({alert$:e}){Wr.default.isSupported()&&new A(t=>{new Wr.default("[data-clipboard-target], [data-clipboard-text]").on("success",r=>t.next(r))}).subscribe(()=>e.next(J("clipboard.copied")))}function Qa(e){if(e.length<2)return e;let[t,r]=e.sort((i,a)=>i.length-a.length).map(i=>i.replace(/[^/]+$/,"")),o=0;if(t===r)o=t.length;else for(;t.charCodeAt(o)===r.charCodeAt(o);)o++;let n=se();return e.map(i=>i.replace(t.slice(0,o),n.base))}function gi({document$:e,location$:t,viewport$:r}){let o=se();if(location.protocol==="file:")return;"scrollRestoration"in history&&(history.scrollRestoration="manual",E(window,"beforeunload").subscribe(()=>{history.scrollRestoration="auto"}));let n=ae("link[rel=icon]");typeof n!="undefined"&&(n.href=n.href);let i=bn(new URL("sitemap.xml",o.base)).pipe(u(l=>Qa(Q("loc",l).map(p=>p.textContent))),O(l=>E(document.body,"click").pipe(M(p=>!p.metaKey&&!p.ctrlKey),O(p=>{if(p.target instanceof Element){let m=p.target.closest("a");if(m&&!m.target){let f=new URL(m.href);if(f.search="",f.hash="",f.pathname!==location.pathname&&l.includes(f.toString()))return p.preventDefault(),R({url:new URL(m.href)})}}return B}))),ne()),a=E(window,"popstate").pipe(M(l=>l.state!==null),u(l=>({url:new URL(location.href),offset:l.state})),ne());P(i,a).pipe(N((l,p)=>l.url.href===p.url.href),u(({url:l})=>l)).subscribe(t);let s=t.pipe(U("pathname"),O(l=>Gt(l.href).pipe(tt(()=>(pn(l),B)))),ne());i.pipe(ot(s)).subscribe(({url:l})=>{history.pushState({},"",`${l}`)});let c=new DOMParser;s.pipe(O(l=>l.text()),u(l=>c.parseFromString(l,"text/html"))).subscribe(e),e.pipe(Kt(1)).subscribe(l=>{for(let p of["title","link[rel=canonical]","meta[name=author]","meta[name=description]","[data-md-component=announce]","[data-md-component=container]","[data-md-component=header-topic]","[data-md-component=logo], .md-logo","[data-md-component=skip]"]){let m=ae(p),f=ae(p,l);typeof m!="undefined"&&typeof f!="undefined"&&De(m,f)}}),e.pipe(Kt(1),u(()=>we("container")),O(l=>R(...Q("script",l))),gr(l=>{let p=nt("script");if(l.src){for(let m of l.getAttributeNames())p.setAttribute(m,l.getAttribute(m));return De(l,p),new A(m=>{p.onload=()=>m.complete()})}else return p.textContent=l.textContent,De(l,p),be})).subscribe(),P(i,a).pipe(ot(e)).subscribe(({url:l,offset:p})=>{l.hash&&!p?mn(l.hash):Hr(p||{y:0})}),r.pipe(Or(i),yr(250),U("offset")).subscribe(({offset:l})=>{history.replaceState(l,"")}),P(i,a).pipe(ve(2,1),M(([l,p])=>l.url.pathname===p.url.pathname),u(([,l])=>l)).subscribe(({offset:l})=>{Hr(l||{y:0})})}var Ja=Qe(Ur());var Si=Qe(Ur());function Vr(e){let t=new RegExp(e.separator,"img"),r=(o,n,i)=>`${n}<mark data-md-highlight>${i}</mark>`;return o=>{o=o.replace(/[\s*+\-:~^]+/g," ").trim();let n=new RegExp(`(^|${e.separator})(${o.replace(/[|\\{}()[\]^$+*?.-]/g,"\\$&").replace(t,"|")})`,"img");return i=>(0,Si.default)(i).replace(n,r).replace(/<\/mark>(\s+)<mark[^>]*>/img,"$1")}}function wi(e){return e.split(/"([^"]+)"/g).map((t,r)=>r&1?t.replace(/^\b|^(?![^\x00-\x7F]|$)|\s+/g," +"):t).join("").replace(/"|(?:^|\s+)[*+\-:^~]+(?=\s+|$)/g,"").trim()}var Ae;(function(n){n[n.SETUP=0]="SETUP",n[n.READY=1]="READY",n[n.QUERY=2]="QUERY",n[n.RESULT=3]="RESULT"})(Ae||(Ae={}));function at(e){return e.type===1}function Ei(e){return e.type===2}function st(e){return e.type===3}function Ya({config:e,docs:t,index:r}){e.lang.length===1&&e.lang[0]==="en"&&(e.lang=[J("search.config.lang")]),e.separator==="[\\s\\-]+"&&(e.separator=J("search.config.separator"));let n={pipeline:J("search.config.pipeline").split(/\s*,\s*/).filter(Boolean),suggestions:Me("search.suggest")};return{config:e,docs:t,index:r,options:n}}function Ti(e,t){let r=se(),o=new Worker(e),n=new T,i=wn(o,{tx$:n}).pipe(u(a=>{if(st(a))for(let s of a.data.items)for(let c of s)c.location=`${new URL(c.location,r.base)}`;return a}),ne());return ge(t).pipe(u(a=>({type:Ae.SETUP,data:Ya(a)}))).subscribe(n.next.bind(n)),{tx$:n,rx$:i}}function Oi(){let e=se();Se(new URL("../versions.json",e.base)).subscribe(t=>{de(".md-header__topic").appendChild(ii(t))})}function Ga(e,{rx$:t}){let r=(__search==null?void 0:__search.transform)||wi,o=rn(e),n=P(E(e,"keyup"),E(e,"focus").pipe(_e(1))).pipe(u(()=>r(e.value)),N()),i=Ie();return i.searchParams.has("q")&&(Fe("search",!0),t.pipe(M(at),me(1)).subscribe(()=>{e.value=i.searchParams.get("q"),xe(e)})),q([n,o]).pipe(u(([a,s])=>({value:a,focus:s})))}function _i(e,{tx$:t,rx$:r}){let o=new T;return o.pipe(U("value"),u(({value:n})=>({type:Ae.QUERY,data:n}))).subscribe(t.next.bind(t)),o.pipe(U("focus")).subscribe(({focus:n})=>{n?(Fe("search",n),$n(e,"")):Wn(e)}),E(e.form,"reset").pipe(Mr(o.pipe(wr(1)))).subscribe(()=>xe(e)),Ga(e,{tx$:t,rx$:r}).pipe(L(o),u(n=>I({ref:e},n)))}function Mi(e,{rx$:t},{query$:r}){let o=new T,n=an(e.parentElement).pipe(M(Boolean)),i=de(":scope > :first-child",e),a=de(":scope > :last-child",e);return t.pipe(M(at),me(1)).subscribe(()=>{Fr(i)}),o.pipe(z(G),pe(r)).subscribe(([{items:c},{value:l}])=>{l?Nn(i,c.length):Fr(i)}),o.pipe(z(G),L(()=>zn(a)),O(({items:c})=>P(R(...c.slice(0,10)),R(...c.slice(10)).pipe(ve(4),kr(n),O(([l])=>R(...l)))))).subscribe(c=>{Dn(a,ri(c))}),t.pipe(M(st),u(({data:c})=>c)).pipe(L(o),u(c=>I({ref:e},c)))}function Xa(e,{query$:t}){return t.pipe(u(({value:r})=>{let o=Ie();return o.hash="",o.searchParams.delete("h"),o.searchParams.set("q",r),{url:o}}))}function Ai(e,t){let r=new T;return r.subscribe(({url:o})=>{e.setAttribute("data-clipboard-text",e.href),e.href=`${o}`}),E(e,"click").subscribe(o=>o.preventDefault()),Xa(e,t).pipe(L(r),u(o=>I({ref:e},o)))}function Li(e,{rx$:t},{keyboard$:r}){let o=new T,n=we("search-query"),i=E(n,"keydown").pipe(z(ke),u(()=>n.value),N());return o.pipe(rt(i),u(([{suggestions:s},c])=>{let l=c.split(/([\s-]+)/);if((s==null?void 0:s.length)&&l[l.length-1]){let p=s[s.length-1];p.startsWith(l[l.length-1])&&(l[l.length-1]=p)}else l.length=0;return l})).subscribe(s=>e.innerHTML=s.join("").replace(/\s/g,"&nbsp;")),r.pipe(M(({mode:s})=>s==="search")).subscribe(s=>{switch(s.type){case"ArrowRight":e.innerText.length&&n.selectionStart===n.value.length&&(n.value=e.innerText);break}}),t.pipe(M(st),u(({data:s})=>s)).pipe(L(o),u(()=>({ref:e})))}function ki(e,{index$:t,keyboard$:r}){let o=se();try{let n=Ti(o.search,t),i=we("search-query",e),a=we("search-result",e),{tx$:s,rx$:c}=n;s.pipe(M(Ei),ot(c.pipe(M(at),me(1)))).subscribe(s.next.bind(s)),r.pipe(M(({mode:m})=>m==="search")).subscribe(m=>{let f=Ne();switch(m.type){case"Enter":if(f===i){let d=new Map;for(let v of Q(":first-child [href]",a)){let h=v.firstElementChild;d.set(v,parseFloat(h.getAttribute("data-md-score")))}if(d.size){let[[v]]=[...d].sort(([,h],[,$])=>$-h);v.click()}m.claim()}break;case"Escape":case"Tab":Fe("search",!1),xe(i,!1);break;case"ArrowUp":case"ArrowDown":if(typeof f=="undefined")xe(i);else{let d=[i,...Q(":not(details) > [href], summary, details[open] [href]",a)],v=Math.max(0,(Math.max(0,d.indexOf(f))+d.length+(m.type==="ArrowUp"?-1:1))%d.length);xe(d[v])}m.claim();break;default:i!==Ne()&&xe(i)}}),r.pipe(M(({mode:m})=>m==="global")).subscribe(m=>{switch(m.type){case"f":case"s":case"/":xe(i),sn(i),m.claim();break}});let l=_i(i,n),p=Mi(a,n,{query$:l});return P(l,p).pipe(Ve(...oe("search-share",e).map(m=>Ai(m,{query$:l})),...oe("search-suggest",e).map(m=>Li(m,n,{keyboard$:r}))))}catch(n){return e.hidden=!0,B}}function Ci(e,{index$:t,location$:r}){return q([t,r.pipe(V(Ie()),M(o=>o.searchParams.has("h")))]).pipe(u(([o,n])=>Vr(o.config)(n.searchParams.get("h"))),u(o=>{var a;let n=new Map,i=document.createNodeIterator(e,NodeFilter.SHOW_TEXT);for(let s=i.nextNode();s;s=i.nextNode())if((a=s.parentElement)==null?void 0:a.offsetHeight){let c=s.textContent,l=o(c);l.length>c.length&&n.set(s,l)}for(let[s,c]of n){let{childNodes:l}=W("span",null,c);s.replaceWith(...Array.from(l))}return{ref:e,nodes:n}}))}function Za(e,{viewport$:t,main$:r}){let o=e.parentElement.offsetTop-e.parentElement.parentElement.offsetTop;return q([r,t]).pipe(u(([{offset:n,height:i},{offset:{y:a}}])=>(i=i+Math.min(o,Math.max(0,a-n))-o,{height:i,locked:a>=n+o})),N((n,i)=>n.height===i.height&&n.locked===i.locked))}function Nr(e,o){var n=o,{header$:t}=n,r=Br(n,["header$"]);let i=new T;return i.pipe(z(G),pe(t)).subscribe({next([{height:a},{height:s}]){Kn(e,a),qn(e,s)},complete(){Qn(e),Bn(e)}}),Za(e,r).pipe(L(i),u(a=>I({ref:e},a)))}function Hi(e,t){if(typeof t!="undefined"){let r=`https://api.github.com/repos/${e}/${t}`;return bt(Se(`${r}/releases/latest`).pipe(u(o=>({version:o.tag_name})),Ue({})),Se(r).pipe(u(o=>({stars:o.stargazers_count,forks:o.forks_count})),Ue({}))).pipe(u(([o,n])=>I(I({},o),n)))}else{let r=`https://api.github.com/repos/${e}`;return Se(r).pipe(u(o=>({repositories:o.public_repos})),Ue({}))}}function ji(e,t){let r=`https://${e}/api/v4/projects/${encodeURIComponent(t)}`;return Se(r).pipe(u(({star_count:o,forks_count:n})=>({stars:o,forks:n})),Ue({}))}function Ri(e){let[t]=e.match(/(git(?:hub|lab))/i)||[];switch(t.toLowerCase()){case"github":let[,r,o]=e.match(/^.+github\.com\/([^/]+)\/?([^/]+)?/i);return Hi(r,o);case"gitlab":let[,n,i]=e.match(/^.+?([^/]*gitlab[^/]+)\/(.+?)\/?$/i);return ji(n,i);default:return B}}var es;function ts(e){return es||(es=Oe(()=>{let t=sessionStorage.getItem(__prefix("__source"));if(t)return R(JSON.parse(t));{let r=Ri(e.href);return r.subscribe(o=>{try{sessionStorage.setItem(__prefix("__source"),JSON.stringify(o))}catch(n){}}),r}}).pipe(tt(()=>B),M(t=>Object.keys(t).length>0),u(t=>({facts:t})),te(1)))}function Fi(e){let t=new T;return t.subscribe(({facts:r})=>{Jn(e,oi(r)),Yn(e,"done")}),ts(e).pipe(L(t),u(r=>I({ref:e},r)))}function rs(e,{viewport$:t,header$:r}){return Re(document.body).pipe(O(()=>Xt(e,{header$:r,viewport$:t})),u(({offset:{y:o}})=>({hidden:o>=10})),U("hidden"))}function Ii(e,t){let r=new T;return r.pipe(z(G)).subscribe({next({hidden:o}){o?Gn(e,"hidden"):Ir(e)},complete(){Ir(e)}}),rs(e,t).pipe(L(r),u(o=>I({ref:e},o)))}function os(e,{viewport$:t,header$:r}){let o=new Map;for(let a of e){let s=decodeURIComponent(a.hash.substring(1)),c=ae(`[id="${s}"]`);typeof c!="undefined"&&o.set(a,c)}let n=r.pipe(u(a=>24+a.height));return Re(document.body).pipe(U("height"),u(()=>{let a=[];return[...o].reduce((s,[c,l])=>{for(;a.length&&o.get(a[a.length-1]).tagName>=l.tagName;)a.pop();let p=l.offsetTop;for(;!p&&l.parentElement;)l=l.parentElement,p=l.offsetTop;return s.set([...a=[...a,c]].reverse(),p)},new Map)}),u(a=>new Map([...a].sort(([,s],[,c])=>s-c))),O(a=>q([n,t]).pipe(Er(([s,c],[l,{offset:{y:p}}])=>{for(;c.length;){let[,m]=c[0];if(m-l<p)s=[...s,c.shift()];else break}for(;s.length;){let[,m]=s[s.length-1];if(m-l>=p)c=[s.pop(),...c];else break}return[s,c]},[[],[...a]]),N((s,c)=>s[0]===c[0]&&s[1]===c[1])))).pipe(u(([a,s])=>({prev:a.map(([c])=>c),next:s.map(([c])=>c)})),V({prev:[],next:[]}),ve(2,1),u(([a,s])=>a.prev.length<s.prev.length?{prev:s.prev.slice(Math.max(0,a.prev.length-1),s.prev.length),next:[]}:{prev:s.prev.slice(-1),next:s.next.slice(0,s.next.length-a.next.length)}))}function Pi(e,t){let r=new T;r.pipe(z(G)).subscribe(({prev:n,next:i})=>{for(let[a]of i)kn(a),An(a);for(let[a,[s]]of n.entries())Ln(s,a===n.length-1),Mn(s,"blur")});let o=Q("[href^=\\#]",e);return os(o,t).pipe(L(r),u(n=>I({ref:e},n)))}function ns(e,{viewport$:t,main$:r}){let o=t.pipe(u(({offset:{y:i}})=>i),ve(2,1),u(([i,a])=>i>a&&a),N()),n=r.pipe(U("active"));return q([n,o]).pipe(u(([{active:i},a])=>({hidden:!(i&&a)})),N((i,a)=>i.hidden===a.hidden))}function $i(e,{viewport$:t,header$:r,main$:o}){let n=new T;return n.pipe(z(G),pe(r.pipe(U("height")))).subscribe({next([{hidden:i},{height:a}]){Zn(e,a+16),i?(Xn(e,"hidden"),xe(e,!1)):Pr(e)},complete(){ei(e),Pr(e)}}),ns(e,{viewport$:t,header$:r,main$:o}).pipe(L(n),u(i=>I({ref:e},i)))}function Wi({document$:e,tablet$:t}){e.pipe(O(()=>R(...Q("[data-md-state=indeterminate]"))),L(r=>{r.indeterminate=!0,r.checked=!1}),ee(r=>E(r,"change").pipe(Ar(()=>r.hasAttribute("data-md-state")),re(r))),pe(t)).subscribe(([r,o])=>{r.removeAttribute("data-md-state"),o&&(r.checked=!1)})}function is(){return/(iPad|iPhone|iPod)/.test(navigator.userAgent)}function Ui({document$:e}){e.pipe(O(()=>R(...Q("[data-md-scrollfix]"))),L(t=>t.removeAttribute("data-md-scrollfix")),M(is),ee(t=>E(t,"touchstart").pipe(re(t)))).subscribe(t=>{let r=t.scrollTop;r===0?t.scrollTop=1:r+t.offsetHeight===t.scrollHeight&&(t.scrollTop=r-1)})}function Vi({viewport$:e,tablet$:t}){q([Yt("search"),t]).pipe(u(([r,o])=>r&&!o),O(r=>R(r).pipe(_e(r?400:100),z(G))),pe(e)).subscribe(([r,{offset:{y:o}}])=>{r?On(document.body,o):_n(document.body)})}document.documentElement.classList.remove("no-js");document.documentElement.classList.add("js");var ct=tn(),er=un(),Dr=dn(),zr=ln(),ue=Sn(),tr=vt("(min-width: 960px)"),Ni=vt("(min-width: 1220px)"),Di=hn(),zi=se(),qi=document.forms.namedItem("search")?(__search==null?void 0:__search.index)||Se(new URL("search/search_index.json",zi.base)):B,qr=new T;xi({alert$:qr});Me("navigation.instant")&&gi({document$:ct,location$:er,viewport$:ue});var Ki;((Ki=zi.version)==null?void 0:Ki.provider)==="mike"&&Oi();P(er,Dr).pipe(_e(125)).subscribe(()=>{Fe("drawer",!1),Fe("search",!1)});zr.pipe(M(({mode:e})=>e==="global")).subscribe(e=>{switch(e.type){case"p":case",":let t=ae("[href][rel=prev]");typeof t!="undefined"&&t.click();break;case"n":case".":let r=ae("[href][rel=next]");typeof r!="undefined"&&r.click();break}});Wi({document$:ct,tablet$:tr});Ui({document$:ct});Vi({viewport$:ue,tablet$:tr});var Pe=mi(we("header"),{viewport$:ue}),rr=ct.pipe(u(()=>we("main")),O(e=>bi(e,{viewport$:ue,header$:Pe})),te(1)),as=P(...oe("dialog").map(e=>fi(e,{alert$:qr})),...oe("header").map(e=>di(e,{viewport$:ue,header$:Pe,main$:rr})),...oe("palette").map(e=>vi(e)),...oe("search").map(e=>ki(e,{index$:qi,keyboard$:zr})),...oe("source").map(e=>Fi(e))),ss=Oe(()=>P(...oe("content").map(e=>ui(e,{target$:Dr,viewport$:ue,print$:Di})),...oe("content").map(e=>Me("search.highlight")?Ci(e,{index$:qi,location$:er}):B),...oe("header-title").map(e=>hi(e,{viewport$:ue,header$:Pe})),...oe("sidebar").map(e=>e.getAttribute("data-md-type")==="navigation"?Cr(Ni,()=>Nr(e,{viewport$:ue,header$:Pe,main$:rr})):Cr(tr,()=>Nr(e,{viewport$:ue,header$:Pe,main$:rr}))),...oe("tabs").map(e=>Ii(e,{viewport$:ue,header$:Pe})),...oe("toc").map(e=>Pi(e,{viewport$:ue,header$:Pe})),...oe("top").map(e=>$i(e,{viewport$:ue,header$:Pe,main$:rr})))),Qi=ct.pipe(O(()=>ss),Ve(as),te(1));Qi.subscribe();window.document$=ct;window.location$=er;window.target$=Dr;window.keyboard$=zr;window.viewport$=ue;window.tablet$=tr;window.screen$=Ni;window.print$=Di;window.alert$=qr;window.component$=Qi;})();
-//# sourceMappingURL=bundle.febc23d1.min.js.map
-
diff --git a/assets/javascripts/bundle.febc23d1.min.js.map b/assets/javascripts/bundle.febc23d1.min.js.map
deleted file mode 100644
index 7a3d8007a..000000000
--- a/assets/javascripts/bundle.febc23d1.min.js.map
+++ /dev/null
@@ -1,7 +0,0 @@
-{
-  "version": 3,
-  "sources": ["node_modules/focus-visible/dist/focus-visible.js", "node_modules/rxjs/node_modules/tslib/tslib.js", "node_modules/clipboard/dist/clipboard.js", "node_modules/escape-html/index.js", "src/assets/javascripts/bundle.ts", "node_modules/rxjs/node_modules/tslib/modules/index.js", "node_modules/rxjs/src/internal/util/isFunction.ts", "node_modules/rxjs/src/internal/util/createErrorClass.ts", "node_modules/rxjs/src/internal/util/UnsubscriptionError.ts", "node_modules/rxjs/src/internal/util/arrRemove.ts", "node_modules/rxjs/src/internal/Subscription.ts", "node_modules/rxjs/src/internal/config.ts", "node_modules/rxjs/src/internal/scheduler/timeoutProvider.ts", "node_modules/rxjs/src/internal/util/reportUnhandledError.ts", "node_modules/rxjs/src/internal/util/noop.ts", "node_modules/rxjs/src/internal/NotificationFactories.ts", "node_modules/rxjs/src/internal/util/errorContext.ts", "node_modules/rxjs/src/internal/Subscriber.ts", "node_modules/rxjs/src/internal/symbol/observable.ts", "node_modules/rxjs/src/internal/util/identity.ts", "node_modules/rxjs/src/internal/util/pipe.ts", "node_modules/rxjs/src/internal/Observable.ts", "node_modules/rxjs/src/internal/util/lift.ts", "node_modules/rxjs/src/internal/operators/OperatorSubscriber.ts", "node_modules/rxjs/src/internal/scheduler/animationFrameProvider.ts", "node_modules/rxjs/src/internal/util/ObjectUnsubscribedError.ts", "node_modules/rxjs/src/internal/Subject.ts", "node_modules/rxjs/src/internal/scheduler/dateTimestampProvider.ts", "node_modules/rxjs/src/internal/ReplaySubject.ts", "node_modules/rxjs/src/internal/scheduler/Action.ts", "node_modules/rxjs/src/internal/scheduler/intervalProvider.ts", "node_modules/rxjs/src/internal/scheduler/AsyncAction.ts", "node_modules/rxjs/src/internal/Scheduler.ts", "node_modules/rxjs/src/internal/scheduler/AsyncScheduler.ts", "node_modules/rxjs/src/internal/scheduler/async.ts", "node_modules/rxjs/src/internal/scheduler/AnimationFrameAction.ts", "node_modules/rxjs/src/internal/scheduler/AnimationFrameScheduler.ts", "node_modules/rxjs/src/internal/scheduler/animationFrame.ts", "node_modules/rxjs/src/internal/observable/empty.ts", "node_modules/rxjs/src/internal/scheduled/scheduleArray.ts", "node_modules/rxjs/src/internal/util/isArrayLike.ts", "node_modules/rxjs/src/internal/util/isPromise.ts", "node_modules/rxjs/src/internal/scheduled/scheduleObservable.ts", "node_modules/rxjs/src/internal/scheduled/schedulePromise.ts", "node_modules/rxjs/src/internal/symbol/iterator.ts", "node_modules/rxjs/src/internal/util/caughtSchedule.ts", "node_modules/rxjs/src/internal/scheduled/scheduleIterable.ts", "node_modules/rxjs/src/internal/scheduled/scheduleAsyncIterable.ts", "node_modules/rxjs/src/internal/util/isInteropObservable.ts", "node_modules/rxjs/src/internal/util/isIterable.ts", "node_modules/rxjs/src/internal/util/isAsyncIterable.ts", "node_modules/rxjs/src/internal/util/throwUnobservableError.ts", "node_modules/rxjs/src/internal/util/isReadableStreamLike.ts", "node_modules/rxjs/src/internal/scheduled/scheduleReadableStreamLike.ts", "node_modules/rxjs/src/internal/scheduled/scheduled.ts", "node_modules/rxjs/src/internal/observable/from.ts", "node_modules/rxjs/src/internal/observable/fromArray.ts", "node_modules/rxjs/src/internal/util/isScheduler.ts", "node_modules/rxjs/src/internal/util/args.ts", "node_modules/rxjs/src/internal/observable/of.ts", "node_modules/rxjs/src/internal/util/isDate.ts", "node_modules/rxjs/src/internal/operators/map.ts", "node_modules/rxjs/src/internal/util/mapOneOrManyArgs.ts", "node_modules/rxjs/src/internal/operators/observeOn.ts", "node_modules/rxjs/src/internal/util/argsArgArrayOrObject.ts", "node_modules/rxjs/src/internal/util/createObject.ts", "node_modules/rxjs/src/internal/observable/combineLatest.ts", "node_modules/rxjs/src/internal/operators/mergeInternals.ts", "node_modules/rxjs/src/internal/operators/mergeMap.ts", "node_modules/rxjs/src/internal/operators/mergeAll.ts", "node_modules/rxjs/src/internal/operators/concatAll.ts", "node_modules/rxjs/src/internal/observable/concat.ts", "node_modules/rxjs/src/internal/observable/defer.ts", "node_modules/rxjs/src/internal/observable/fromEvent.ts", "node_modules/rxjs/src/internal/observable/fromEventPattern.ts", "node_modules/rxjs/src/internal/observable/timer.ts", "node_modules/rxjs/src/internal/observable/merge.ts", "node_modules/rxjs/src/internal/observable/never.ts", "node_modules/rxjs/src/internal/util/argsOrArgArray.ts", "node_modules/rxjs/src/internal/operators/filter.ts", "node_modules/rxjs/src/internal/observable/zip.ts", "node_modules/rxjs/src/internal/operators/bufferCount.ts", "node_modules/rxjs/src/internal/operators/catchError.ts", "node_modules/rxjs/src/internal/operators/scanInternals.ts", "node_modules/rxjs/src/internal/operators/combineLatest.ts", "node_modules/rxjs/src/internal/operators/combineLatestWith.ts", "node_modules/rxjs/src/internal/operators/concatMap.ts", "node_modules/rxjs/src/internal/operators/debounceTime.ts", "node_modules/rxjs/src/internal/operators/defaultIfEmpty.ts", "node_modules/rxjs/src/internal/operators/take.ts", "node_modules/rxjs/src/internal/operators/ignoreElements.ts", "node_modules/rxjs/src/internal/operators/mapTo.ts", "node_modules/rxjs/src/internal/operators/delayWhen.ts", "node_modules/rxjs/src/internal/operators/delay.ts", "node_modules/rxjs/src/internal/operators/distinctUntilChanged.ts", "node_modules/rxjs/src/internal/operators/distinctUntilKeyChanged.ts", "node_modules/rxjs/src/internal/operators/finalize.ts", "node_modules/rxjs/src/internal/operators/takeLast.ts", "node_modules/rxjs/src/internal/operators/merge.ts", "node_modules/rxjs/src/internal/operators/mergeWith.ts", "node_modules/rxjs/src/internal/operators/sample.ts", "node_modules/rxjs/src/internal/operators/scan.ts", "node_modules/rxjs/src/internal/operators/share.ts", "node_modules/rxjs/src/internal/operators/shareReplay.ts", "node_modules/rxjs/src/internal/operators/skip.ts", "node_modules/rxjs/src/internal/operators/skipUntil.ts", "node_modules/rxjs/src/internal/operators/startWith.ts", "node_modules/rxjs/src/internal/operators/switchMap.ts", "node_modules/rxjs/src/internal/operators/switchMapTo.ts", "node_modules/rxjs/src/internal/operators/takeUntil.ts", "node_modules/rxjs/src/internal/operators/takeWhile.ts", "node_modules/rxjs/src/internal/operators/tap.ts", "node_modules/rxjs/src/internal/operators/throttle.ts", "node_modules/rxjs/src/internal/operators/withLatestFrom.ts", "node_modules/rxjs/src/internal/operators/zip.ts", "node_modules/rxjs/src/internal/operators/zipWith.ts", "src/assets/javascripts/browser/document/index.ts", "src/assets/javascripts/browser/element/_/index.ts", "src/assets/javascripts/browser/element/focus/index.ts", "src/assets/javascripts/browser/element/size/index.ts", "src/assets/javascripts/browser/element/offset/index.ts", "src/assets/javascripts/browser/element/selection/index.ts", "src/assets/javascripts/browser/toggle/index.ts", "src/assets/javascripts/browser/keyboard/index.ts", "src/assets/javascripts/browser/location/_/index.ts", "src/assets/javascripts/browser/location/hash/index.ts", "src/assets/javascripts/browser/media/index.ts", "src/assets/javascripts/browser/request/index.ts", "src/assets/javascripts/browser/viewport/offset/index.ts", "src/assets/javascripts/browser/viewport/size/index.ts", "src/assets/javascripts/browser/viewport/_/index.ts", "src/assets/javascripts/browser/worker/index.ts", "src/assets/javascripts/_/index.ts", "src/assets/javascripts/components/_/index.ts", "src/assets/javascripts/components/content/code/index.ts", "src/assets/javascripts/actions/_/index.ts", "src/assets/javascripts/actions/anchor/index.ts", "src/assets/javascripts/actions/dialog/index.ts", "src/assets/javascripts/actions/header/_/index.ts", "src/assets/javascripts/actions/header/title/index.ts", "src/assets/javascripts/actions/search/query/index.ts", "src/assets/javascripts/utilities/h/index.ts", "src/assets/javascripts/utilities/string/index.ts", "src/assets/javascripts/actions/search/result/index.ts", "src/assets/javascripts/actions/sidebar/index.ts", "src/assets/javascripts/actions/source/index.ts", "src/assets/javascripts/actions/tabs/index.ts", "src/assets/javascripts/actions/top/index.ts", "src/assets/javascripts/templates/clipboard/index.tsx", "src/assets/javascripts/templates/search/index.tsx", "src/assets/javascripts/templates/source/index.tsx", "src/assets/javascripts/templates/table/index.tsx", "src/assets/javascripts/templates/version/index.tsx", "src/assets/javascripts/components/content/details/index.ts", "src/assets/javascripts/components/content/table/index.ts", "src/assets/javascripts/components/content/_/index.ts", "src/assets/javascripts/components/dialog/index.ts", "src/assets/javascripts/components/header/_/index.ts", "src/assets/javascripts/components/header/title/index.ts", "src/assets/javascripts/components/main/index.ts", "src/assets/javascripts/components/palette/index.ts", "src/assets/javascripts/integrations/clipboard/index.ts", "src/assets/javascripts/integrations/instant/index.ts", "src/assets/javascripts/integrations/search/document/index.ts", "src/assets/javascripts/integrations/search/highlighter/index.ts", "src/assets/javascripts/integrations/search/query/transform/index.ts", "src/assets/javascripts/integrations/search/worker/message/index.ts", "src/assets/javascripts/integrations/search/worker/_/index.ts", "src/assets/javascripts/integrations/version/index.ts", "src/assets/javascripts/components/search/query/index.ts", "src/assets/javascripts/components/search/result/index.ts", "src/assets/javascripts/components/search/share/index.ts", "src/assets/javascripts/components/search/suggest/index.ts", "src/assets/javascripts/components/search/_/index.ts", "src/assets/javascripts/components/search/highlight/index.ts", "src/assets/javascripts/components/sidebar/index.ts", "src/assets/javascripts/components/source/facts/github/index.ts", "src/assets/javascripts/components/source/facts/gitlab/index.ts", "src/assets/javascripts/components/source/facts/_/index.ts", "src/assets/javascripts/components/source/_/index.ts", "src/assets/javascripts/components/tabs/index.ts", "src/assets/javascripts/components/toc/index.ts", "src/assets/javascripts/components/top/index.ts", "src/assets/javascripts/patches/indeterminate/index.ts", "src/assets/javascripts/patches/scrollfix/index.ts", "src/assets/javascripts/patches/scrolllock/index.ts"],
-  "sourcesContent": ["(function (global, factory) {\n  typeof exports === 'object' && typeof module !== 'undefined' ? factory() :\n  typeof define === 'function' && define.amd ? define(factory) :\n  (factory());\n}(this, (function () { 'use strict';\n\n  /**\n   * Applies the :focus-visible polyfill at the given scope.\n   * A scope in this case is either the top-level Document or a Shadow Root.\n   *\n   * @param {(Document|ShadowRoot)} scope\n   * @see https://github.com/WICG/focus-visible\n   */\n  function applyFocusVisiblePolyfill(scope) {\n    var hadKeyboardEvent = true;\n    var hadFocusVisibleRecently = false;\n    var hadFocusVisibleRecentlyTimeout = null;\n\n    var inputTypesAllowlist = {\n      text: true,\n      search: true,\n      url: true,\n      tel: true,\n      email: true,\n      password: true,\n      number: true,\n      date: true,\n      month: true,\n      week: true,\n      time: true,\n      datetime: true,\n      'datetime-local': true\n    };\n\n    /**\n     * Helper function for legacy browsers and iframes which sometimes focus\n     * elements like document, body, and non-interactive SVG.\n     * @param {Element} el\n     */\n    function isValidFocusTarget(el) {\n      if (\n        el &&\n        el !== document &&\n        el.nodeName !== 'HTML' &&\n        el.nodeName !== 'BODY' &&\n        'classList' in el &&\n        'contains' in el.classList\n      ) {\n        return true;\n      }\n      return false;\n    }\n\n    /**\n     * Computes whether the given element should automatically trigger the\n     * `focus-visible` class being added, i.e. whether it should always match\n     * `:focus-visible` when focused.\n     * @param {Element} el\n     * @return {boolean}\n     */\n    function focusTriggersKeyboardModality(el) {\n      var type = el.type;\n      var tagName = el.tagName;\n\n      if (tagName === 'INPUT' && inputTypesAllowlist[type] && !el.readOnly) {\n        return true;\n      }\n\n      if (tagName === 'TEXTAREA' && !el.readOnly) {\n        return true;\n      }\n\n      if (el.isContentEditable) {\n        return true;\n      }\n\n      return false;\n    }\n\n    /**\n     * Add the `focus-visible` class to the given element if it was not added by\n     * the author.\n     * @param {Element} el\n     */\n    function addFocusVisibleClass(el) {\n      if (el.classList.contains('focus-visible')) {\n        return;\n      }\n      el.classList.add('focus-visible');\n      el.setAttribute('data-focus-visible-added', '');\n    }\n\n    /**\n     * Remove the `focus-visible` class from the given element if it was not\n     * originally added by the author.\n     * @param {Element} el\n     */\n    function removeFocusVisibleClass(el) {\n      if (!el.hasAttribute('data-focus-visible-added')) {\n        return;\n      }\n      el.classList.remove('focus-visible');\n      el.removeAttribute('data-focus-visible-added');\n    }\n\n    /**\n     * If the most recent user interaction was via the keyboard;\n     * and the key press did not include a meta, alt/option, or control key;\n     * then the modality is keyboard. Otherwise, the modality is not keyboard.\n     * Apply `focus-visible` to any current active element and keep track\n     * of our keyboard modality state with `hadKeyboardEvent`.\n     * @param {KeyboardEvent} e\n     */\n    function onKeyDown(e) {\n      if (e.metaKey || e.altKey || e.ctrlKey) {\n        return;\n      }\n\n      if (isValidFocusTarget(scope.activeElement)) {\n        addFocusVisibleClass(scope.activeElement);\n      }\n\n      hadKeyboardEvent = true;\n    }\n\n    /**\n     * If at any point a user clicks with a pointing device, ensure that we change\n     * the modality away from keyboard.\n     * This avoids the situation where a user presses a key on an already focused\n     * element, and then clicks on a different element, focusing it with a\n     * pointing device, while we still think we're in keyboard modality.\n     * @param {Event} e\n     */\n    function onPointerDown(e) {\n      hadKeyboardEvent = false;\n    }\n\n    /**\n     * On `focus`, add the `focus-visible` class to the target if:\n     * - the target received focus as a result of keyboard navigation, or\n     * - the event target is an element that will likely require interaction\n     *   via the keyboard (e.g. a text box)\n     * @param {Event} e\n     */\n    function onFocus(e) {\n      // Prevent IE from focusing the document or HTML element.\n      if (!isValidFocusTarget(e.target)) {\n        return;\n      }\n\n      if (hadKeyboardEvent || focusTriggersKeyboardModality(e.target)) {\n        addFocusVisibleClass(e.target);\n      }\n    }\n\n    /**\n     * On `blur`, remove the `focus-visible` class from the target.\n     * @param {Event} e\n     */\n    function onBlur(e) {\n      if (!isValidFocusTarget(e.target)) {\n        return;\n      }\n\n      if (\n        e.target.classList.contains('focus-visible') ||\n        e.target.hasAttribute('data-focus-visible-added')\n      ) {\n        // To detect a tab/window switch, we look for a blur event followed\n        // rapidly by a visibility change.\n        // If we don't see a visibility change within 100ms, it's probably a\n        // regular focus change.\n        hadFocusVisibleRecently = true;\n        window.clearTimeout(hadFocusVisibleRecentlyTimeout);\n        hadFocusVisibleRecentlyTimeout = window.setTimeout(function() {\n          hadFocusVisibleRecently = false;\n        }, 100);\n        removeFocusVisibleClass(e.target);\n      }\n    }\n\n    /**\n     * If the user changes tabs, keep track of whether or not the previously\n     * focused element had .focus-visible.\n     * @param {Event} e\n     */\n    function onVisibilityChange(e) {\n      if (document.visibilityState === 'hidden') {\n        // If the tab becomes active again, the browser will handle calling focus\n        // on the element (Safari actually calls it twice).\n        // If this tab change caused a blur on an element with focus-visible,\n        // re-apply the class when the user switches back to the tab.\n        if (hadFocusVisibleRecently) {\n          hadKeyboardEvent = true;\n        }\n        addInitialPointerMoveListeners();\n      }\n    }\n\n    /**\n     * Add a group of listeners to detect usage of any pointing devices.\n     * These listeners will be added when the polyfill first loads, and anytime\n     * the window is blurred, so that they are active when the window regains\n     * focus.\n     */\n    function addInitialPointerMoveListeners() {\n      document.addEventListener('mousemove', onInitialPointerMove);\n      document.addEventListener('mousedown', onInitialPointerMove);\n      document.addEventListener('mouseup', onInitialPointerMove);\n      document.addEventListener('pointermove', onInitialPointerMove);\n      document.addEventListener('pointerdown', onInitialPointerMove);\n      document.addEventListener('pointerup', onInitialPointerMove);\n      document.addEventListener('touchmove', onInitialPointerMove);\n      document.addEventListener('touchstart', onInitialPointerMove);\n      document.addEventListener('touchend', onInitialPointerMove);\n    }\n\n    function removeInitialPointerMoveListeners() {\n      document.removeEventListener('mousemove', onInitialPointerMove);\n      document.removeEventListener('mousedown', onInitialPointerMove);\n      document.removeEventListener('mouseup', onInitialPointerMove);\n      document.removeEventListener('pointermove', onInitialPointerMove);\n      document.removeEventListener('pointerdown', onInitialPointerMove);\n      document.removeEventListener('pointerup', onInitialPointerMove);\n      document.removeEventListener('touchmove', onInitialPointerMove);\n      document.removeEventListener('touchstart', onInitialPointerMove);\n      document.removeEventListener('touchend', onInitialPointerMove);\n    }\n\n    /**\n     * When the polfyill first loads, assume the user is in keyboard modality.\n     * If any event is received from a pointing device (e.g. mouse, pointer,\n     * touch), turn off keyboard modality.\n     * This accounts for situations where focus enters the page from the URL bar.\n     * @param {Event} e\n     */\n    function onInitialPointerMove(e) {\n      // Work around a Safari quirk that fires a mousemove on <html> whenever the\n      // window blurs, even if you're tabbing out of the page. \u00AF\\_(\u30C4)_/\u00AF\n      if (e.target.nodeName && e.target.nodeName.toLowerCase() === 'html') {\n        return;\n      }\n\n      hadKeyboardEvent = false;\n      removeInitialPointerMoveListeners();\n    }\n\n    // For some kinds of state, we are interested in changes at the global scope\n    // only. For example, global pointer input, global key presses and global\n    // visibility change should affect the state at every scope:\n    document.addEventListener('keydown', onKeyDown, true);\n    document.addEventListener('mousedown', onPointerDown, true);\n    document.addEventListener('pointerdown', onPointerDown, true);\n    document.addEventListener('touchstart', onPointerDown, true);\n    document.addEventListener('visibilitychange', onVisibilityChange, true);\n\n    addInitialPointerMoveListeners();\n\n    // For focus and blur, we specifically care about state changes in the local\n    // scope. This is because focus / blur events that originate from within a\n    // shadow root are not re-dispatched from the host element if it was already\n    // the active element in its own scope:\n    scope.addEventListener('focus', onFocus, true);\n    scope.addEventListener('blur', onBlur, true);\n\n    // We detect that a node is a ShadowRoot by ensuring that it is a\n    // DocumentFragment and also has a host property. This check covers native\n    // implementation and polyfill implementation transparently. If we only cared\n    // about the native implementation, we could just check if the scope was\n    // an instance of a ShadowRoot.\n    if (scope.nodeType === Node.DOCUMENT_FRAGMENT_NODE && scope.host) {\n      // Since a ShadowRoot is a special kind of DocumentFragment, it does not\n      // have a root element to add a class to. So, we add this attribute to the\n      // host element instead:\n      scope.host.setAttribute('data-js-focus-visible', '');\n    } else if (scope.nodeType === Node.DOCUMENT_NODE) {\n      document.documentElement.classList.add('js-focus-visible');\n      document.documentElement.setAttribute('data-js-focus-visible', '');\n    }\n  }\n\n  // It is important to wrap all references to global window and document in\n  // these checks to support server-side rendering use cases\n  // @see https://github.com/WICG/focus-visible/issues/199\n  if (typeof window !== 'undefined' && typeof document !== 'undefined') {\n    // Make the polyfill helper globally available. This can be used as a signal\n    // to interested libraries that wish to coordinate with the polyfill for e.g.,\n    // applying the polyfill to a shadow root:\n    window.applyFocusVisiblePolyfill = applyFocusVisiblePolyfill;\n\n    // Notify interested libraries of the polyfill's presence, in case the\n    // polyfill was loaded lazily:\n    var event;\n\n    try {\n      event = new CustomEvent('focus-visible-polyfill-ready');\n    } catch (error) {\n      // IE11 does not support using CustomEvent as a constructor directly:\n      event = document.createEvent('CustomEvent');\n      event.initCustomEvent('focus-visible-polyfill-ready', false, false, {});\n    }\n\n    window.dispatchEvent(event);\n  }\n\n  if (typeof document !== 'undefined') {\n    // Apply the polyfill to the global document, so that no JavaScript\n    // coordination is required to use the polyfill in the top-level document:\n    applyFocusVisiblePolyfill(document);\n  }\n\n})));\n", "/*! *****************************************************************************\r\nCopyright (c) Microsoft Corporation.\r\n\r\nPermission to use, copy, modify, and/or distribute this software for any\r\npurpose with or without fee is hereby granted.\r\n\r\nTHE SOFTWARE IS PROVIDED \"AS IS\" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH\r\nREGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY\r\nAND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,\r\nINDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM\r\nLOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR\r\nOTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR\r\nPERFORMANCE OF THIS SOFTWARE.\r\n***************************************************************************** */\r\n/* global global, define, System, Reflect, Promise */\r\nvar __extends;\r\nvar __assign;\r\nvar __rest;\r\nvar __decorate;\r\nvar __param;\r\nvar __metadata;\r\nvar __awaiter;\r\nvar __generator;\r\nvar __exportStar;\r\nvar __values;\r\nvar __read;\r\nvar __spread;\r\nvar __spreadArrays;\r\nvar __spreadArray;\r\nvar __await;\r\nvar __asyncGenerator;\r\nvar __asyncDelegator;\r\nvar __asyncValues;\r\nvar __makeTemplateObject;\r\nvar __importStar;\r\nvar __importDefault;\r\nvar __classPrivateFieldGet;\r\nvar __classPrivateFieldSet;\r\nvar __createBinding;\r\n(function (factory) {\r\n    var root = typeof global === \"object\" ? global : typeof self === \"object\" ? self : typeof this === \"object\" ? this : {};\r\n    if (typeof define === \"function\" && define.amd) {\r\n        define(\"tslib\", [\"exports\"], function (exports) { factory(createExporter(root, createExporter(exports))); });\r\n    }\r\n    else if (typeof module === \"object\" && typeof module.exports === \"object\") {\r\n        factory(createExporter(root, createExporter(module.exports)));\r\n    }\r\n    else {\r\n        factory(createExporter(root));\r\n    }\r\n    function createExporter(exports, previous) {\r\n        if (exports !== root) {\r\n            if (typeof Object.create === \"function\") {\r\n                Object.defineProperty(exports, \"__esModule\", { value: true });\r\n            }\r\n            else {\r\n                exports.__esModule = true;\r\n            }\r\n        }\r\n        return function (id, v) { return exports[id] = previous ? previous(id, v) : v; };\r\n    }\r\n})\r\n(function (exporter) {\r\n    var extendStatics = Object.setPrototypeOf ||\r\n        ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||\r\n        function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };\r\n\r\n    __extends = function (d, b) {\r\n        if (typeof b !== \"function\" && b !== null)\r\n            throw new TypeError(\"Class extends value \" + String(b) + \" is not a constructor or null\");\r\n        extendStatics(d, b);\r\n        function __() { this.constructor = d; }\r\n        d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());\r\n    };\r\n\r\n    __assign = Object.assign || function (t) {\r\n        for (var s, i = 1, n = arguments.length; i < n; i++) {\r\n            s = arguments[i];\r\n            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) t[p] = s[p];\r\n        }\r\n        return t;\r\n    };\r\n\r\n    __rest = function (s, e) {\r\n        var t = {};\r\n        for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)\r\n            t[p] = s[p];\r\n        if (s != null && typeof Object.getOwnPropertySymbols === \"function\")\r\n            for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {\r\n                if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))\r\n                    t[p[i]] = s[p[i]];\r\n            }\r\n        return t;\r\n    };\r\n\r\n    __decorate = function (decorators, target, key, desc) {\r\n        var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;\r\n        if (typeof Reflect === \"object\" && typeof Reflect.decorate === \"function\") r = Reflect.decorate(decorators, target, key, desc);\r\n        else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;\r\n        return c > 3 && r && Object.defineProperty(target, key, r), r;\r\n    };\r\n\r\n    __param = function (paramIndex, decorator) {\r\n        return function (target, key) { decorator(target, key, paramIndex); }\r\n    };\r\n\r\n    __metadata = function (metadataKey, metadataValue) {\r\n        if (typeof Reflect === \"object\" && typeof Reflect.metadata === \"function\") return Reflect.metadata(metadataKey, metadataValue);\r\n    };\r\n\r\n    __awaiter = function (thisArg, _arguments, P, generator) {\r\n        function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }\r\n        return new (P || (P = Promise))(function (resolve, reject) {\r\n            function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }\r\n            function rejected(value) { try { step(generator[\"throw\"](value)); } catch (e) { reject(e); } }\r\n            function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }\r\n            step((generator = generator.apply(thisArg, _arguments || [])).next());\r\n        });\r\n    };\r\n\r\n    __generator = function (thisArg, body) {\r\n        var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;\r\n        return g = { next: verb(0), \"throw\": verb(1), \"return\": verb(2) }, typeof Symbol === \"function\" && (g[Symbol.iterator] = function() { return this; }), g;\r\n        function verb(n) { return function (v) { return step([n, v]); }; }\r\n        function step(op) {\r\n            if (f) throw new TypeError(\"Generator is already executing.\");\r\n            while (_) try {\r\n                if (f = 1, y && (t = op[0] & 2 ? y[\"return\"] : op[0] ? y[\"throw\"] || ((t = y[\"return\"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;\r\n                if (y = 0, t) op = [op[0] & 2, t.value];\r\n                switch (op[0]) {\r\n                    case 0: case 1: t = op; break;\r\n                    case 4: _.label++; return { value: op[1], done: false };\r\n                    case 5: _.label++; y = op[1]; op = [0]; continue;\r\n                    case 7: op = _.ops.pop(); _.trys.pop(); continue;\r\n                    default:\r\n                        if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }\r\n                        if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }\r\n                        if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }\r\n                        if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }\r\n                        if (t[2]) _.ops.pop();\r\n                        _.trys.pop(); continue;\r\n                }\r\n                op = body.call(thisArg, _);\r\n            } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }\r\n            if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };\r\n        }\r\n    };\r\n\r\n    __exportStar = function(m, o) {\r\n        for (var p in m) if (p !== \"default\" && !Object.prototype.hasOwnProperty.call(o, p)) __createBinding(o, m, p);\r\n    };\r\n\r\n    __createBinding = Object.create ? (function(o, m, k, k2) {\r\n        if (k2 === undefined) k2 = k;\r\n        Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });\r\n    }) : (function(o, m, k, k2) {\r\n        if (k2 === undefined) k2 = k;\r\n        o[k2] = m[k];\r\n    });\r\n\r\n    __values = function (o) {\r\n        var s = typeof Symbol === \"function\" && Symbol.iterator, m = s && o[s], i = 0;\r\n        if (m) return m.call(o);\r\n        if (o && typeof o.length === \"number\") return {\r\n            next: function () {\r\n                if (o && i >= o.length) o = void 0;\r\n                return { value: o && o[i++], done: !o };\r\n            }\r\n        };\r\n        throw new TypeError(s ? \"Object is not iterable.\" : \"Symbol.iterator is not defined.\");\r\n    };\r\n\r\n    __read = function (o, n) {\r\n        var m = typeof Symbol === \"function\" && o[Symbol.iterator];\r\n        if (!m) return o;\r\n        var i = m.call(o), r, ar = [], e;\r\n        try {\r\n            while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);\r\n        }\r\n        catch (error) { e = { error: error }; }\r\n        finally {\r\n            try {\r\n                if (r && !r.done && (m = i[\"return\"])) m.call(i);\r\n            }\r\n            finally { if (e) throw e.error; }\r\n        }\r\n        return ar;\r\n    };\r\n\r\n    /** @deprecated */\r\n    __spread = function () {\r\n        for (var ar = [], i = 0; i < arguments.length; i++)\r\n            ar = ar.concat(__read(arguments[i]));\r\n        return ar;\r\n    };\r\n\r\n    /** @deprecated */\r\n    __spreadArrays = function () {\r\n        for (var s = 0, i = 0, il = arguments.length; i < il; i++) s += arguments[i].length;\r\n        for (var r = Array(s), k = 0, i = 0; i < il; i++)\r\n            for (var a = arguments[i], j = 0, jl = a.length; j < jl; j++, k++)\r\n                r[k] = a[j];\r\n        return r;\r\n    };\r\n\r\n    __spreadArray = function (to, from) {\r\n        for (var i = 0, il = from.length, j = to.length; i < il; i++, j++)\r\n            to[j] = from[i];\r\n        return to;\r\n    };\r\n\r\n    __await = function (v) {\r\n        return this instanceof __await ? (this.v = v, this) : new __await(v);\r\n    };\r\n\r\n    __asyncGenerator = function (thisArg, _arguments, generator) {\r\n        if (!Symbol.asyncIterator) throw new TypeError(\"Symbol.asyncIterator is not defined.\");\r\n        var g = generator.apply(thisArg, _arguments || []), i, q = [];\r\n        return i = {}, verb(\"next\"), verb(\"throw\"), verb(\"return\"), i[Symbol.asyncIterator] = function () { return this; }, i;\r\n        function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; }\r\n        function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }\r\n        function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r);  }\r\n        function fulfill(value) { resume(\"next\", value); }\r\n        function reject(value) { resume(\"throw\", value); }\r\n        function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }\r\n    };\r\n\r\n    __asyncDelegator = function (o) {\r\n        var i, p;\r\n        return i = {}, verb(\"next\"), verb(\"throw\", function (e) { throw e; }), verb(\"return\"), i[Symbol.iterator] = function () { return this; }, i;\r\n        function verb(n, f) { i[n] = o[n] ? function (v) { return (p = !p) ? { value: __await(o[n](v)), done: n === \"return\" } : f ? f(v) : v; } : f; }\r\n    };\r\n\r\n    __asyncValues = function (o) {\r\n        if (!Symbol.asyncIterator) throw new TypeError(\"Symbol.asyncIterator is not defined.\");\r\n        var m = o[Symbol.asyncIterator], i;\r\n        return m ? m.call(o) : (o = typeof __values === \"function\" ? __values(o) : o[Symbol.iterator](), i = {}, verb(\"next\"), verb(\"throw\"), verb(\"return\"), i[Symbol.asyncIterator] = function () { return this; }, i);\r\n        function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }\r\n        function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }\r\n    };\r\n\r\n    __makeTemplateObject = function (cooked, raw) {\r\n        if (Object.defineProperty) { Object.defineProperty(cooked, \"raw\", { value: raw }); } else { cooked.raw = raw; }\r\n        return cooked;\r\n    };\r\n\r\n    var __setModuleDefault = Object.create ? (function(o, v) {\r\n        Object.defineProperty(o, \"default\", { enumerable: true, value: v });\r\n    }) : function(o, v) {\r\n        o[\"default\"] = v;\r\n    };\r\n\r\n    __importStar = function (mod) {\r\n        if (mod && mod.__esModule) return mod;\r\n        var result = {};\r\n        if (mod != null) for (var k in mod) if (k !== \"default\" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);\r\n        __setModuleDefault(result, mod);\r\n        return result;\r\n    };\r\n\r\n    __importDefault = function (mod) {\r\n        return (mod && mod.__esModule) ? mod : { \"default\": mod };\r\n    };\r\n\r\n    __classPrivateFieldGet = function (receiver, privateMap) {\r\n        if (!privateMap.has(receiver)) {\r\n            throw new TypeError(\"attempted to get private field on non-instance\");\r\n        }\r\n        return privateMap.get(receiver);\r\n    };\r\n\r\n    __classPrivateFieldSet = function (receiver, privateMap, value) {\r\n        if (!privateMap.has(receiver)) {\r\n            throw new TypeError(\"attempted to set private field on non-instance\");\r\n        }\r\n        privateMap.set(receiver, value);\r\n        return value;\r\n    };\r\n\r\n    exporter(\"__extends\", __extends);\r\n    exporter(\"__assign\", __assign);\r\n    exporter(\"__rest\", __rest);\r\n    exporter(\"__decorate\", __decorate);\r\n    exporter(\"__param\", __param);\r\n    exporter(\"__metadata\", __metadata);\r\n    exporter(\"__awaiter\", __awaiter);\r\n    exporter(\"__generator\", __generator);\r\n    exporter(\"__exportStar\", __exportStar);\r\n    exporter(\"__createBinding\", __createBinding);\r\n    exporter(\"__values\", __values);\r\n    exporter(\"__read\", __read);\r\n    exporter(\"__spread\", __spread);\r\n    exporter(\"__spreadArrays\", __spreadArrays);\r\n    exporter(\"__spreadArray\", __spreadArray);\r\n    exporter(\"__await\", __await);\r\n    exporter(\"__asyncGenerator\", __asyncGenerator);\r\n    exporter(\"__asyncDelegator\", __asyncDelegator);\r\n    exporter(\"__asyncValues\", __asyncValues);\r\n    exporter(\"__makeTemplateObject\", __makeTemplateObject);\r\n    exporter(\"__importStar\", __importStar);\r\n    exporter(\"__importDefault\", __importDefault);\r\n    exporter(\"__classPrivateFieldGet\", __classPrivateFieldGet);\r\n    exporter(\"__classPrivateFieldSet\", __classPrivateFieldSet);\r\n});\r\n", "/*!\n * clipboard.js v2.0.8\n * https://clipboardjs.com/\n *\n * Licensed MIT \u00A9 Zeno Rocha\n */\n(function webpackUniversalModuleDefinition(root, factory) {\n\tif(typeof exports === 'object' && typeof module === 'object')\n\t\tmodule.exports = factory();\n\telse if(typeof define === 'function' && define.amd)\n\t\tdefine([], factory);\n\telse if(typeof exports === 'object')\n\t\texports[\"ClipboardJS\"] = factory();\n\telse\n\t\troot[\"ClipboardJS\"] = factory();\n})(this, function() {\nreturn /******/ (function() { // webpackBootstrap\n/******/ \tvar __webpack_modules__ = ({\n\n/***/ 134:\n/***/ (function(__unused_webpack_module, __webpack_exports__, __webpack_require__) {\n\n\"use strict\";\n\n// EXPORTS\n__webpack_require__.d(__webpack_exports__, {\n  \"default\": function() { return /* binding */ clipboard; }\n});\n\n// EXTERNAL MODULE: ./node_modules/tiny-emitter/index.js\nvar tiny_emitter = __webpack_require__(279);\nvar tiny_emitter_default = /*#__PURE__*/__webpack_require__.n(tiny_emitter);\n// EXTERNAL MODULE: ./node_modules/good-listener/src/listen.js\nvar listen = __webpack_require__(370);\nvar listen_default = /*#__PURE__*/__webpack_require__.n(listen);\n// EXTERNAL MODULE: ./node_modules/select/src/select.js\nvar src_select = __webpack_require__(817);\nvar select_default = /*#__PURE__*/__webpack_require__.n(src_select);\n;// CONCATENATED MODULE: ./src/clipboard-action.js\nfunction _typeof(obj) { \"@babel/helpers - typeof\"; if (typeof Symbol === \"function\" && typeof Symbol.iterator === \"symbol\") { _typeof = function _typeof(obj) { return typeof obj; }; } else { _typeof = function _typeof(obj) { return obj && typeof Symbol === \"function\" && obj.constructor === Symbol && obj !== Symbol.prototype ? \"symbol\" : typeof obj; }; } return _typeof(obj); }\n\nfunction _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError(\"Cannot call a class as a function\"); } }\n\nfunction _defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if (\"value\" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } }\n\nfunction _createClass(Constructor, protoProps, staticProps) { if (protoProps) _defineProperties(Constructor.prototype, protoProps); if (staticProps) _defineProperties(Constructor, staticProps); return Constructor; }\n\n\n/**\n * Inner class which performs selection from either `text` or `target`\n * properties and then executes copy or cut operations.\n */\n\nvar ClipboardAction = /*#__PURE__*/function () {\n  /**\n   * @param {Object} options\n   */\n  function ClipboardAction(options) {\n    _classCallCheck(this, ClipboardAction);\n\n    this.resolveOptions(options);\n    this.initSelection();\n  }\n  /**\n   * Defines base properties passed from constructor.\n   * @param {Object} options\n   */\n\n\n  _createClass(ClipboardAction, [{\n    key: \"resolveOptions\",\n    value: function resolveOptions() {\n      var options = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};\n      this.action = options.action;\n      this.container = options.container;\n      this.emitter = options.emitter;\n      this.target = options.target;\n      this.text = options.text;\n      this.trigger = options.trigger;\n      this.selectedText = '';\n    }\n    /**\n     * Decides which selection strategy is going to be applied based\n     * on the existence of `text` and `target` properties.\n     */\n\n  }, {\n    key: \"initSelection\",\n    value: function initSelection() {\n      if (this.text) {\n        this.selectFake();\n      } else if (this.target) {\n        this.selectTarget();\n      }\n    }\n    /**\n     * Creates a fake textarea element, sets its value from `text` property,\n     */\n\n  }, {\n    key: \"createFakeElement\",\n    value: function createFakeElement() {\n      var isRTL = document.documentElement.getAttribute('dir') === 'rtl';\n      this.fakeElem = document.createElement('textarea'); // Prevent zooming on iOS\n\n      this.fakeElem.style.fontSize = '12pt'; // Reset box model\n\n      this.fakeElem.style.border = '0';\n      this.fakeElem.style.padding = '0';\n      this.fakeElem.style.margin = '0'; // Move element out of screen horizontally\n\n      this.fakeElem.style.position = 'absolute';\n      this.fakeElem.style[isRTL ? 'right' : 'left'] = '-9999px'; // Move element to the same position vertically\n\n      var yPosition = window.pageYOffset || document.documentElement.scrollTop;\n      this.fakeElem.style.top = \"\".concat(yPosition, \"px\");\n      this.fakeElem.setAttribute('readonly', '');\n      this.fakeElem.value = this.text;\n      return this.fakeElem;\n    }\n    /**\n     * Get's the value of fakeElem,\n     * and makes a selection on it.\n     */\n\n  }, {\n    key: \"selectFake\",\n    value: function selectFake() {\n      var _this = this;\n\n      var fakeElem = this.createFakeElement();\n\n      this.fakeHandlerCallback = function () {\n        return _this.removeFake();\n      };\n\n      this.fakeHandler = this.container.addEventListener('click', this.fakeHandlerCallback) || true;\n      this.container.appendChild(fakeElem);\n      this.selectedText = select_default()(fakeElem);\n      this.copyText();\n      this.removeFake();\n    }\n    /**\n     * Only removes the fake element after another click event, that way\n     * a user can hit `Ctrl+C` to copy because selection still exists.\n     */\n\n  }, {\n    key: \"removeFake\",\n    value: function removeFake() {\n      if (this.fakeHandler) {\n        this.container.removeEventListener('click', this.fakeHandlerCallback);\n        this.fakeHandler = null;\n        this.fakeHandlerCallback = null;\n      }\n\n      if (this.fakeElem) {\n        this.container.removeChild(this.fakeElem);\n        this.fakeElem = null;\n      }\n    }\n    /**\n     * Selects the content from element passed on `target` property.\n     */\n\n  }, {\n    key: \"selectTarget\",\n    value: function selectTarget() {\n      this.selectedText = select_default()(this.target);\n      this.copyText();\n    }\n    /**\n     * Executes the copy operation based on the current selection.\n     */\n\n  }, {\n    key: \"copyText\",\n    value: function copyText() {\n      var succeeded;\n\n      try {\n        succeeded = document.execCommand(this.action);\n      } catch (err) {\n        succeeded = false;\n      }\n\n      this.handleResult(succeeded);\n    }\n    /**\n     * Fires an event based on the copy operation result.\n     * @param {Boolean} succeeded\n     */\n\n  }, {\n    key: \"handleResult\",\n    value: function handleResult(succeeded) {\n      this.emitter.emit(succeeded ? 'success' : 'error', {\n        action: this.action,\n        text: this.selectedText,\n        trigger: this.trigger,\n        clearSelection: this.clearSelection.bind(this)\n      });\n    }\n    /**\n     * Moves focus away from `target` and back to the trigger, removes current selection.\n     */\n\n  }, {\n    key: \"clearSelection\",\n    value: function clearSelection() {\n      if (this.trigger) {\n        this.trigger.focus();\n      }\n\n      document.activeElement.blur();\n      window.getSelection().removeAllRanges();\n    }\n    /**\n     * Sets the `action` to be performed which can be either 'copy' or 'cut'.\n     * @param {String} action\n     */\n\n  }, {\n    key: \"destroy\",\n\n    /**\n     * Destroy lifecycle.\n     */\n    value: function destroy() {\n      this.removeFake();\n    }\n  }, {\n    key: \"action\",\n    set: function set() {\n      var action = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 'copy';\n      this._action = action;\n\n      if (this._action !== 'copy' && this._action !== 'cut') {\n        throw new Error('Invalid \"action\" value, use either \"copy\" or \"cut\"');\n      }\n    }\n    /**\n     * Gets the `action` property.\n     * @return {String}\n     */\n    ,\n    get: function get() {\n      return this._action;\n    }\n    /**\n     * Sets the `target` property using an element\n     * that will be have its content copied.\n     * @param {Element} target\n     */\n\n  }, {\n    key: \"target\",\n    set: function set(target) {\n      if (target !== undefined) {\n        if (target && _typeof(target) === 'object' && target.nodeType === 1) {\n          if (this.action === 'copy' && target.hasAttribute('disabled')) {\n            throw new Error('Invalid \"target\" attribute. Please use \"readonly\" instead of \"disabled\" attribute');\n          }\n\n          if (this.action === 'cut' && (target.hasAttribute('readonly') || target.hasAttribute('disabled'))) {\n            throw new Error('Invalid \"target\" attribute. You can\\'t cut text from elements with \"readonly\" or \"disabled\" attributes');\n          }\n\n          this._target = target;\n        } else {\n          throw new Error('Invalid \"target\" value, use a valid Element');\n        }\n      }\n    }\n    /**\n     * Gets the `target` property.\n     * @return {String|HTMLElement}\n     */\n    ,\n    get: function get() {\n      return this._target;\n    }\n  }]);\n\n  return ClipboardAction;\n}();\n\n/* harmony default export */ var clipboard_action = (ClipboardAction);\n;// CONCATENATED MODULE: ./src/clipboard.js\nfunction clipboard_typeof(obj) { \"@babel/helpers - typeof\"; if (typeof Symbol === \"function\" && typeof Symbol.iterator === \"symbol\") { clipboard_typeof = function _typeof(obj) { return typeof obj; }; } else { clipboard_typeof = function _typeof(obj) { return obj && typeof Symbol === \"function\" && obj.constructor === Symbol && obj !== Symbol.prototype ? \"symbol\" : typeof obj; }; } return clipboard_typeof(obj); }\n\nfunction clipboard_classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError(\"Cannot call a class as a function\"); } }\n\nfunction clipboard_defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if (\"value\" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } }\n\nfunction clipboard_createClass(Constructor, protoProps, staticProps) { if (protoProps) clipboard_defineProperties(Constructor.prototype, protoProps); if (staticProps) clipboard_defineProperties(Constructor, staticProps); return Constructor; }\n\nfunction _inherits(subClass, superClass) { if (typeof superClass !== \"function\" && superClass !== null) { throw new TypeError(\"Super expression must either be null or a function\"); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, writable: true, configurable: true } }); if (superClass) _setPrototypeOf(subClass, superClass); }\n\nfunction _setPrototypeOf(o, p) { _setPrototypeOf = Object.setPrototypeOf || function _setPrototypeOf(o, p) { o.__proto__ = p; return o; }; return _setPrototypeOf(o, p); }\n\nfunction _createSuper(Derived) { var hasNativeReflectConstruct = _isNativeReflectConstruct(); return function _createSuperInternal() { var Super = _getPrototypeOf(Derived), result; if (hasNativeReflectConstruct) { var NewTarget = _getPrototypeOf(this).constructor; result = Reflect.construct(Super, arguments, NewTarget); } else { result = Super.apply(this, arguments); } return _possibleConstructorReturn(this, result); }; }\n\nfunction _possibleConstructorReturn(self, call) { if (call && (clipboard_typeof(call) === \"object\" || typeof call === \"function\")) { return call; } return _assertThisInitialized(self); }\n\nfunction _assertThisInitialized(self) { if (self === void 0) { throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\"); } return self; }\n\nfunction _isNativeReflectConstruct() { if (typeof Reflect === \"undefined\" || !Reflect.construct) return false; if (Reflect.construct.sham) return false; if (typeof Proxy === \"function\") return true; try { Date.prototype.toString.call(Reflect.construct(Date, [], function () {})); return true; } catch (e) { return false; } }\n\nfunction _getPrototypeOf(o) { _getPrototypeOf = Object.setPrototypeOf ? Object.getPrototypeOf : function _getPrototypeOf(o) { return o.__proto__ || Object.getPrototypeOf(o); }; return _getPrototypeOf(o); }\n\n\n\n\n/**\n * Helper function to retrieve attribute value.\n * @param {String} suffix\n * @param {Element} element\n */\n\nfunction getAttributeValue(suffix, element) {\n  var attribute = \"data-clipboard-\".concat(suffix);\n\n  if (!element.hasAttribute(attribute)) {\n    return;\n  }\n\n  return element.getAttribute(attribute);\n}\n/**\n * Base class which takes one or more elements, adds event listeners to them,\n * and instantiates a new `ClipboardAction` on each click.\n */\n\n\nvar Clipboard = /*#__PURE__*/function (_Emitter) {\n  _inherits(Clipboard, _Emitter);\n\n  var _super = _createSuper(Clipboard);\n\n  /**\n   * @param {String|HTMLElement|HTMLCollection|NodeList} trigger\n   * @param {Object} options\n   */\n  function Clipboard(trigger, options) {\n    var _this;\n\n    clipboard_classCallCheck(this, Clipboard);\n\n    _this = _super.call(this);\n\n    _this.resolveOptions(options);\n\n    _this.listenClick(trigger);\n\n    return _this;\n  }\n  /**\n   * Defines if attributes would be resolved using internal setter functions\n   * or custom functions that were passed in the constructor.\n   * @param {Object} options\n   */\n\n\n  clipboard_createClass(Clipboard, [{\n    key: \"resolveOptions\",\n    value: function resolveOptions() {\n      var options = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};\n      this.action = typeof options.action === 'function' ? options.action : this.defaultAction;\n      this.target = typeof options.target === 'function' ? options.target : this.defaultTarget;\n      this.text = typeof options.text === 'function' ? options.text : this.defaultText;\n      this.container = clipboard_typeof(options.container) === 'object' ? options.container : document.body;\n    }\n    /**\n     * Adds a click event listener to the passed trigger.\n     * @param {String|HTMLElement|HTMLCollection|NodeList} trigger\n     */\n\n  }, {\n    key: \"listenClick\",\n    value: function listenClick(trigger) {\n      var _this2 = this;\n\n      this.listener = listen_default()(trigger, 'click', function (e) {\n        return _this2.onClick(e);\n      });\n    }\n    /**\n     * Defines a new `ClipboardAction` on each click event.\n     * @param {Event} e\n     */\n\n  }, {\n    key: \"onClick\",\n    value: function onClick(e) {\n      var trigger = e.delegateTarget || e.currentTarget;\n\n      if (this.clipboardAction) {\n        this.clipboardAction = null;\n      }\n\n      this.clipboardAction = new clipboard_action({\n        action: this.action(trigger),\n        target: this.target(trigger),\n        text: this.text(trigger),\n        container: this.container,\n        trigger: trigger,\n        emitter: this\n      });\n    }\n    /**\n     * Default `action` lookup function.\n     * @param {Element} trigger\n     */\n\n  }, {\n    key: \"defaultAction\",\n    value: function defaultAction(trigger) {\n      return getAttributeValue('action', trigger);\n    }\n    /**\n     * Default `target` lookup function.\n     * @param {Element} trigger\n     */\n\n  }, {\n    key: \"defaultTarget\",\n    value: function defaultTarget(trigger) {\n      var selector = getAttributeValue('target', trigger);\n\n      if (selector) {\n        return document.querySelector(selector);\n      }\n    }\n    /**\n     * Returns the support of the given action, or all actions if no action is\n     * given.\n     * @param {String} [action]\n     */\n\n  }, {\n    key: \"defaultText\",\n\n    /**\n     * Default `text` lookup function.\n     * @param {Element} trigger\n     */\n    value: function defaultText(trigger) {\n      return getAttributeValue('text', trigger);\n    }\n    /**\n     * Destroy lifecycle.\n     */\n\n  }, {\n    key: \"destroy\",\n    value: function destroy() {\n      this.listener.destroy();\n\n      if (this.clipboardAction) {\n        this.clipboardAction.destroy();\n        this.clipboardAction = null;\n      }\n    }\n  }], [{\n    key: \"isSupported\",\n    value: function isSupported() {\n      var action = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : ['copy', 'cut'];\n      var actions = typeof action === 'string' ? [action] : action;\n      var support = !!document.queryCommandSupported;\n      actions.forEach(function (action) {\n        support = support && !!document.queryCommandSupported(action);\n      });\n      return support;\n    }\n  }]);\n\n  return Clipboard;\n}((tiny_emitter_default()));\n\n/* harmony default export */ var clipboard = (Clipboard);\n\n/***/ }),\n\n/***/ 828:\n/***/ (function(module) {\n\nvar DOCUMENT_NODE_TYPE = 9;\n\n/**\n * A polyfill for Element.matches()\n */\nif (typeof Element !== 'undefined' && !Element.prototype.matches) {\n    var proto = Element.prototype;\n\n    proto.matches = proto.matchesSelector ||\n                    proto.mozMatchesSelector ||\n                    proto.msMatchesSelector ||\n                    proto.oMatchesSelector ||\n                    proto.webkitMatchesSelector;\n}\n\n/**\n * Finds the closest parent that matches a selector.\n *\n * @param {Element} element\n * @param {String} selector\n * @return {Function}\n */\nfunction closest (element, selector) {\n    while (element && element.nodeType !== DOCUMENT_NODE_TYPE) {\n        if (typeof element.matches === 'function' &&\n            element.matches(selector)) {\n          return element;\n        }\n        element = element.parentNode;\n    }\n}\n\nmodule.exports = closest;\n\n\n/***/ }),\n\n/***/ 438:\n/***/ (function(module, __unused_webpack_exports, __webpack_require__) {\n\nvar closest = __webpack_require__(828);\n\n/**\n * Delegates event to a selector.\n *\n * @param {Element} element\n * @param {String} selector\n * @param {String} type\n * @param {Function} callback\n * @param {Boolean} useCapture\n * @return {Object}\n */\nfunction _delegate(element, selector, type, callback, useCapture) {\n    var listenerFn = listener.apply(this, arguments);\n\n    element.addEventListener(type, listenerFn, useCapture);\n\n    return {\n        destroy: function() {\n            element.removeEventListener(type, listenerFn, useCapture);\n        }\n    }\n}\n\n/**\n * Delegates event to a selector.\n *\n * @param {Element|String|Array} [elements]\n * @param {String} selector\n * @param {String} type\n * @param {Function} callback\n * @param {Boolean} useCapture\n * @return {Object}\n */\nfunction delegate(elements, selector, type, callback, useCapture) {\n    // Handle the regular Element usage\n    if (typeof elements.addEventListener === 'function') {\n        return _delegate.apply(null, arguments);\n    }\n\n    // Handle Element-less usage, it defaults to global delegation\n    if (typeof type === 'function') {\n        // Use `document` as the first parameter, then apply arguments\n        // This is a short way to .unshift `arguments` without running into deoptimizations\n        return _delegate.bind(null, document).apply(null, arguments);\n    }\n\n    // Handle Selector-based usage\n    if (typeof elements === 'string') {\n        elements = document.querySelectorAll(elements);\n    }\n\n    // Handle Array-like based usage\n    return Array.prototype.map.call(elements, function (element) {\n        return _delegate(element, selector, type, callback, useCapture);\n    });\n}\n\n/**\n * Finds closest match and invokes callback.\n *\n * @param {Element} element\n * @param {String} selector\n * @param {String} type\n * @param {Function} callback\n * @return {Function}\n */\nfunction listener(element, selector, type, callback) {\n    return function(e) {\n        e.delegateTarget = closest(e.target, selector);\n\n        if (e.delegateTarget) {\n            callback.call(element, e);\n        }\n    }\n}\n\nmodule.exports = delegate;\n\n\n/***/ }),\n\n/***/ 879:\n/***/ (function(__unused_webpack_module, exports) {\n\n/**\n * Check if argument is a HTML element.\n *\n * @param {Object} value\n * @return {Boolean}\n */\nexports.node = function(value) {\n    return value !== undefined\n        && value instanceof HTMLElement\n        && value.nodeType === 1;\n};\n\n/**\n * Check if argument is a list of HTML elements.\n *\n * @param {Object} value\n * @return {Boolean}\n */\nexports.nodeList = function(value) {\n    var type = Object.prototype.toString.call(value);\n\n    return value !== undefined\n        && (type === '[object NodeList]' || type === '[object HTMLCollection]')\n        && ('length' in value)\n        && (value.length === 0 || exports.node(value[0]));\n};\n\n/**\n * Check if argument is a string.\n *\n * @param {Object} value\n * @return {Boolean}\n */\nexports.string = function(value) {\n    return typeof value === 'string'\n        || value instanceof String;\n};\n\n/**\n * Check if argument is a function.\n *\n * @param {Object} value\n * @return {Boolean}\n */\nexports.fn = function(value) {\n    var type = Object.prototype.toString.call(value);\n\n    return type === '[object Function]';\n};\n\n\n/***/ }),\n\n/***/ 370:\n/***/ (function(module, __unused_webpack_exports, __webpack_require__) {\n\nvar is = __webpack_require__(879);\nvar delegate = __webpack_require__(438);\n\n/**\n * Validates all params and calls the right\n * listener function based on its target type.\n *\n * @param {String|HTMLElement|HTMLCollection|NodeList} target\n * @param {String} type\n * @param {Function} callback\n * @return {Object}\n */\nfunction listen(target, type, callback) {\n    if (!target && !type && !callback) {\n        throw new Error('Missing required arguments');\n    }\n\n    if (!is.string(type)) {\n        throw new TypeError('Second argument must be a String');\n    }\n\n    if (!is.fn(callback)) {\n        throw new TypeError('Third argument must be a Function');\n    }\n\n    if (is.node(target)) {\n        return listenNode(target, type, callback);\n    }\n    else if (is.nodeList(target)) {\n        return listenNodeList(target, type, callback);\n    }\n    else if (is.string(target)) {\n        return listenSelector(target, type, callback);\n    }\n    else {\n        throw new TypeError('First argument must be a String, HTMLElement, HTMLCollection, or NodeList');\n    }\n}\n\n/**\n * Adds an event listener to a HTML element\n * and returns a remove listener function.\n *\n * @param {HTMLElement} node\n * @param {String} type\n * @param {Function} callback\n * @return {Object}\n */\nfunction listenNode(node, type, callback) {\n    node.addEventListener(type, callback);\n\n    return {\n        destroy: function() {\n            node.removeEventListener(type, callback);\n        }\n    }\n}\n\n/**\n * Add an event listener to a list of HTML elements\n * and returns a remove listener function.\n *\n * @param {NodeList|HTMLCollection} nodeList\n * @param {String} type\n * @param {Function} callback\n * @return {Object}\n */\nfunction listenNodeList(nodeList, type, callback) {\n    Array.prototype.forEach.call(nodeList, function(node) {\n        node.addEventListener(type, callback);\n    });\n\n    return {\n        destroy: function() {\n            Array.prototype.forEach.call(nodeList, function(node) {\n                node.removeEventListener(type, callback);\n            });\n        }\n    }\n}\n\n/**\n * Add an event listener to a selector\n * and returns a remove listener function.\n *\n * @param {String} selector\n * @param {String} type\n * @param {Function} callback\n * @return {Object}\n */\nfunction listenSelector(selector, type, callback) {\n    return delegate(document.body, selector, type, callback);\n}\n\nmodule.exports = listen;\n\n\n/***/ }),\n\n/***/ 817:\n/***/ (function(module) {\n\nfunction select(element) {\n    var selectedText;\n\n    if (element.nodeName === 'SELECT') {\n        element.focus();\n\n        selectedText = element.value;\n    }\n    else if (element.nodeName === 'INPUT' || element.nodeName === 'TEXTAREA') {\n        var isReadOnly = element.hasAttribute('readonly');\n\n        if (!isReadOnly) {\n            element.setAttribute('readonly', '');\n        }\n\n        element.select();\n        element.setSelectionRange(0, element.value.length);\n\n        if (!isReadOnly) {\n            element.removeAttribute('readonly');\n        }\n\n        selectedText = element.value;\n    }\n    else {\n        if (element.hasAttribute('contenteditable')) {\n            element.focus();\n        }\n\n        var selection = window.getSelection();\n        var range = document.createRange();\n\n        range.selectNodeContents(element);\n        selection.removeAllRanges();\n        selection.addRange(range);\n\n        selectedText = selection.toString();\n    }\n\n    return selectedText;\n}\n\nmodule.exports = select;\n\n\n/***/ }),\n\n/***/ 279:\n/***/ (function(module) {\n\nfunction E () {\n  // Keep this empty so it's easier to inherit from\n  // (via https://github.com/lipsmack from https://github.com/scottcorgan/tiny-emitter/issues/3)\n}\n\nE.prototype = {\n  on: function (name, callback, ctx) {\n    var e = this.e || (this.e = {});\n\n    (e[name] || (e[name] = [])).push({\n      fn: callback,\n      ctx: ctx\n    });\n\n    return this;\n  },\n\n  once: function (name, callback, ctx) {\n    var self = this;\n    function listener () {\n      self.off(name, listener);\n      callback.apply(ctx, arguments);\n    };\n\n    listener._ = callback\n    return this.on(name, listener, ctx);\n  },\n\n  emit: function (name) {\n    var data = [].slice.call(arguments, 1);\n    var evtArr = ((this.e || (this.e = {}))[name] || []).slice();\n    var i = 0;\n    var len = evtArr.length;\n\n    for (i; i < len; i++) {\n      evtArr[i].fn.apply(evtArr[i].ctx, data);\n    }\n\n    return this;\n  },\n\n  off: function (name, callback) {\n    var e = this.e || (this.e = {});\n    var evts = e[name];\n    var liveEvents = [];\n\n    if (evts && callback) {\n      for (var i = 0, len = evts.length; i < len; i++) {\n        if (evts[i].fn !== callback && evts[i].fn._ !== callback)\n          liveEvents.push(evts[i]);\n      }\n    }\n\n    // Remove event from queue to prevent memory leak\n    // Suggested by https://github.com/lazd\n    // Ref: https://github.com/scottcorgan/tiny-emitter/commit/c6ebfaa9bc973b33d110a84a307742b7cf94c953#commitcomment-5024910\n\n    (liveEvents.length)\n      ? e[name] = liveEvents\n      : delete e[name];\n\n    return this;\n  }\n};\n\nmodule.exports = E;\nmodule.exports.TinyEmitter = E;\n\n\n/***/ })\n\n/******/ \t});\n/************************************************************************/\n/******/ \t// The module cache\n/******/ \tvar __webpack_module_cache__ = {};\n/******/ \t\n/******/ \t// The require function\n/******/ \tfunction __webpack_require__(moduleId) {\n/******/ \t\t// Check if module is in cache\n/******/ \t\tif(__webpack_module_cache__[moduleId]) {\n/******/ \t\t\treturn __webpack_module_cache__[moduleId].exports;\n/******/ \t\t}\n/******/ \t\t// Create a new module (and put it into the cache)\n/******/ \t\tvar module = __webpack_module_cache__[moduleId] = {\n/******/ \t\t\t// no module.id needed\n/******/ \t\t\t// no module.loaded needed\n/******/ \t\t\texports: {}\n/******/ \t\t};\n/******/ \t\n/******/ \t\t// Execute the module function\n/******/ \t\t__webpack_modules__[moduleId](module, module.exports, __webpack_require__);\n/******/ \t\n/******/ \t\t// Return the exports of the module\n/******/ \t\treturn module.exports;\n/******/ \t}\n/******/ \t\n/************************************************************************/\n/******/ \t/* webpack/runtime/compat get default export */\n/******/ \t!function() {\n/******/ \t\t// getDefaultExport function for compatibility with non-harmony modules\n/******/ \t\t__webpack_require__.n = function(module) {\n/******/ \t\t\tvar getter = module && module.__esModule ?\n/******/ \t\t\t\tfunction() { return module['default']; } :\n/******/ \t\t\t\tfunction() { return module; };\n/******/ \t\t\t__webpack_require__.d(getter, { a: getter });\n/******/ \t\t\treturn getter;\n/******/ \t\t};\n/******/ \t}();\n/******/ \t\n/******/ \t/* webpack/runtime/define property getters */\n/******/ \t!function() {\n/******/ \t\t// define getter functions for harmony exports\n/******/ \t\t__webpack_require__.d = function(exports, definition) {\n/******/ \t\t\tfor(var key in definition) {\n/******/ \t\t\t\tif(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n/******/ \t\t\t\t\tObject.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n/******/ \t\t\t\t}\n/******/ \t\t\t}\n/******/ \t\t};\n/******/ \t}();\n/******/ \t\n/******/ \t/* webpack/runtime/hasOwnProperty shorthand */\n/******/ \t!function() {\n/******/ \t\t__webpack_require__.o = function(obj, prop) { return Object.prototype.hasOwnProperty.call(obj, prop); }\n/******/ \t}();\n/******/ \t\n/************************************************************************/\n/******/ \t// module exports must be returned from runtime so entry inlining is disabled\n/******/ \t// startup\n/******/ \t// Load entry module and return exports\n/******/ \treturn __webpack_require__(134);\n/******/ })()\n.default;\n});", "/*!\n * escape-html\n * Copyright(c) 2012-2013 TJ Holowaychuk\n * Copyright(c) 2015 Andreas Lubbe\n * Copyright(c) 2015 Tiancheng \"Timothy\" Gu\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module variables.\n * @private\n */\n\nvar matchHtmlRegExp = /[\"'&<>]/;\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = escapeHtml;\n\n/**\n * Escape special characters in the given string of html.\n *\n * @param  {string} string The string to escape for inserting into HTML\n * @return {string}\n * @public\n */\n\nfunction escapeHtml(string) {\n  var str = '' + string;\n  var match = matchHtmlRegExp.exec(str);\n\n  if (!match) {\n    return str;\n  }\n\n  var escape;\n  var html = '';\n  var index = 0;\n  var lastIndex = 0;\n\n  for (index = match.index; index < str.length; index++) {\n    switch (str.charCodeAt(index)) {\n      case 34: // \"\n        escape = '&quot;';\n        break;\n      case 38: // &\n        escape = '&amp;';\n        break;\n      case 39: // '\n        escape = '&#39;';\n        break;\n      case 60: // <\n        escape = '&lt;';\n        break;\n      case 62: // >\n        escape = '&gt;';\n        break;\n      default:\n        continue;\n    }\n\n    if (lastIndex !== index) {\n      html += str.substring(lastIndex, index);\n    }\n\n    lastIndex = index + 1;\n    html += escape;\n  }\n\n  return lastIndex !== index\n    ? html + str.substring(lastIndex, index)\n    : html;\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport \"focus-visible\"\nimport { NEVER, Subject, defer, merge } from \"rxjs\"\nimport {\n  delay,\n  filter,\n  map,\n  mergeWith,\n  shareReplay,\n  switchMap\n} from \"rxjs/operators\"\n\nimport { configuration, feature } from \"./_\"\nimport {\n  at,\n  getElement,\n  requestJSON,\n  setToggle,\n  watchDocument,\n  watchKeyboard,\n  watchLocation,\n  watchLocationTarget,\n  watchMedia,\n  watchPrint,\n  watchViewport\n} from \"./browser\"\nimport {\n  getComponentElement,\n  getComponentElements,\n  mountBackToTop,\n  mountContent,\n  mountDialog,\n  mountHeader,\n  mountHeaderTitle,\n  mountPalette,\n  mountSearch,\n  mountSearchHiglight,\n  mountSidebar,\n  mountSource,\n  mountTableOfContents,\n  mountTabs,\n  watchHeader,\n  watchMain\n} from \"./components\"\nimport {\n  SearchIndex,\n  setupClipboardJS,\n  setupInstantLoading,\n  setupVersionSelector\n} from \"./integrations\"\nimport {\n  patchIndeterminate,\n  patchScrollfix,\n  patchScrolllock\n} from \"./patches\"\n\n/* ----------------------------------------------------------------------------\n * Application\n * ------------------------------------------------------------------------- */\n\n/* Yay, JavaScript is available */\ndocument.documentElement.classList.remove(\"no-js\")\ndocument.documentElement.classList.add(\"js\")\n\n/* Set up navigation observables and subjects */\nconst document$ = watchDocument()\nconst location$ = watchLocation()\nconst target$   = watchLocationTarget()\nconst keyboard$ = watchKeyboard()\n\n/* Set up media observables */\nconst viewport$ = watchViewport()\nconst tablet$   = watchMedia(\"(min-width: 960px)\")\nconst screen$   = watchMedia(\"(min-width: 1220px)\")\nconst print$    = watchPrint()\n\n/* Retrieve search index, if search is enabled */\nconst config = configuration()\nconst index$ = document.forms.namedItem(\"search\")\n  ? __search?.index || requestJSON<SearchIndex>(\n    new URL(\"search/search_index.json\", config.base)\n  )\n  : NEVER\n\n/* Set up Clipboard.js integration */\nconst alert$ = new Subject<string>()\nsetupClipboardJS({ alert$ })\n\n/* Set up instant loading, if enabled */\nif (feature(\"navigation.instant\"))\n  setupInstantLoading({ document$, location$, viewport$ })\n\n/* Set up version selector */\nif (config.version?.provider === \"mike\")\n  setupVersionSelector()\n\n/* Always close drawer and search on navigation */\nmerge(location$, target$)\n  .pipe(\n    delay(125)\n  )\n    .subscribe(() => {\n      setToggle(\"drawer\", false)\n      setToggle(\"search\", false)\n    })\n\n/* Set up global keyboard handlers */\nkeyboard$\n  .pipe(\n    filter(({ mode }) => mode === \"global\")\n  )\n    .subscribe(key => {\n      switch (key.type) {\n\n        /* Go to previous page */\n        case \"p\":\n        case \",\":\n          const prev = getElement(\"[href][rel=prev]\")\n          if (typeof prev !== \"undefined\")\n            prev.click()\n          break\n\n        /* Go to next page */\n        case \"n\":\n        case \".\":\n          const next = getElement(\"[href][rel=next]\")\n          if (typeof next !== \"undefined\")\n            next.click()\n          break\n      }\n    })\n\n/* Set up patches */\npatchIndeterminate({ document$, tablet$ })\npatchScrollfix({ document$ })\npatchScrolllock({ viewport$, tablet$ })\n\n/* Set up header and main area observable */\nconst header$ = watchHeader(getComponentElement(\"header\"), { viewport$ })\nconst main$ = document$\n  .pipe(\n    map(() => getComponentElement(\"main\")),\n    switchMap(el => watchMain(el, { viewport$, header$ })),\n    shareReplay(1)\n  )\n\n/* Set up control component observables */\nconst control$ = merge(\n\n  /* Dialog */\n  ...getComponentElements(\"dialog\")\n    .map(el => mountDialog(el, { alert$ })),\n\n  /* Header */\n  ...getComponentElements(\"header\")\n    .map(el => mountHeader(el, { viewport$, header$, main$ })),\n\n  /* Color palette */\n  ...getComponentElements(\"palette\")\n    .map(el => mountPalette(el)),\n\n  /* Search */\n  ...getComponentElements(\"search\")\n    .map(el => mountSearch(el, { index$, keyboard$ })),\n\n  /* Repository information */\n  ...getComponentElements(\"source\")\n    .map(el => mountSource(el))\n)\n\n/* Set up content component observables */\nconst content$ = defer(() => merge(\n\n  /* Content */\n  ...getComponentElements(\"content\")\n    .map(el => mountContent(el, { target$, viewport$, print$ })),\n\n  /* Search highlighting */\n  ...getComponentElements(\"content\")\n    .map(el => feature(\"search.highlight\")\n      ? mountSearchHiglight(el, { index$, location$ })\n      : NEVER\n    ),\n\n  /* Header title */\n  ...getComponentElements(\"header-title\")\n    .map(el => mountHeaderTitle(el, { viewport$, header$ })),\n\n  /* Sidebar */\n  ...getComponentElements(\"sidebar\")\n    .map(el => el.getAttribute(\"data-md-type\") === \"navigation\"\n      ? at(screen$, () => mountSidebar(el, { viewport$, header$, main$ }))\n      : at(tablet$, () => mountSidebar(el, { viewport$, header$, main$ }))\n    ),\n\n  /* Navigation tabs */\n  ...getComponentElements(\"tabs\")\n    .map(el => mountTabs(el, { viewport$, header$ })),\n\n  /* Table of contents */\n  ...getComponentElements(\"toc\")\n    .map(el => mountTableOfContents(el, { viewport$, header$ })),\n\n  /* Back-to-top button */\n  ...getComponentElements(\"top\")\n    .map(el => mountBackToTop(el, { viewport$, header$, main$ }))\n))\n\n/* Set up component observables */\nconst component$ = document$\n  .pipe(\n    switchMap(() => content$),\n    mergeWith(control$),\n    shareReplay(1)\n  )\n\n/* Subscribe to all components */\ncomponent$.subscribe()\n\n/* ----------------------------------------------------------------------------\n * Exports\n * ------------------------------------------------------------------------- */\n\nwindow.document$  = document$          /* Document observable */\nwindow.location$  = location$          /* Location subject */\nwindow.target$    = target$            /* Location target observable */\nwindow.keyboard$  = keyboard$          /* Keyboard observable */\nwindow.viewport$  = viewport$          /* Viewport observable */\nwindow.tablet$    = tablet$            /* Tablet observable */\nwindow.screen$    = screen$            /* Screen observable */\nwindow.print$     = print$             /* Print mode observable */\nwindow.alert$     = alert$             /* Alert subject */\nwindow.component$ = component$         /* Component observable */\n", "import tslib from '../tslib.js';\r\nconst {\r\n    __extends,\r\n    __assign,\r\n    __rest,\r\n    __decorate,\r\n    __param,\r\n    __metadata,\r\n    __awaiter,\r\n    __generator,\r\n    __exportStar,\r\n    __createBinding,\r\n    __values,\r\n    __read,\r\n    __spread,\r\n    __spreadArrays,\r\n    __spreadArray,\r\n    __await,\r\n    __asyncGenerator,\r\n    __asyncDelegator,\r\n    __asyncValues,\r\n    __makeTemplateObject,\r\n    __importStar,\r\n    __importDefault,\r\n    __classPrivateFieldGet,\r\n    __classPrivateFieldSet,\r\n} = tslib;\r\nexport {\r\n    __extends,\r\n    __assign,\r\n    __rest,\r\n    __decorate,\r\n    __param,\r\n    __metadata,\r\n    __awaiter,\r\n    __generator,\r\n    __exportStar,\r\n    __createBinding,\r\n    __values,\r\n    __read,\r\n    __spread,\r\n    __spreadArrays,\r\n    __spreadArray,\r\n    __await,\r\n    __asyncGenerator,\r\n    __asyncDelegator,\r\n    __asyncValues,\r\n    __makeTemplateObject,\r\n    __importStar,\r\n    __importDefault,\r\n    __classPrivateFieldGet,\r\n    __classPrivateFieldSet,\r\n};\r\n", null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { ReplaySubject, Subject, fromEvent } from \"rxjs\"\nimport { mapTo } from \"rxjs/operators\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch document\n *\n * Documents are implemented as subjects, so all downstream observables are\n * automatically updated when a new document is emitted.\n *\n * @returns Document subject\n */\nexport function watchDocument(): Subject<Document> {\n  const document$ = new ReplaySubject<Document>()\n  fromEvent(document, \"DOMContentLoaded\")\n    .pipe(\n      mapTo(document)\n    )\n      .subscribe(document$)\n\n  /* Return document */\n  return document$\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve an element matching the query selector\n *\n * @template T - Element type\n *\n * @param selector - Query selector\n * @param node - Node of reference\n *\n * @returns Element or nothing\n */\nexport function getElement<T extends keyof HTMLElementTagNameMap>(\n  selector: T, node?: ParentNode\n): HTMLElementTagNameMap[T]\n\nexport function getElement<T extends HTMLElement>(\n  selector: string, node?: ParentNode\n): T | undefined\n\nexport function getElement<T extends HTMLElement>(\n  selector: string, node: ParentNode = document\n): T | undefined {\n  return node.querySelector<T>(selector) || undefined\n}\n\n/**\n * Retrieve an element matching a query selector or throw a reference error\n *\n * @template T - Element type\n *\n * @param selector - Query selector\n * @param node - Node of reference\n *\n * @returns Element\n */\nexport function getElementOrThrow<T extends keyof HTMLElementTagNameMap>(\n  selector: T, node?: ParentNode\n): HTMLElementTagNameMap[T]\n\nexport function getElementOrThrow<T extends HTMLElement>(\n  selector: string, node?: ParentNode\n): T\n\nexport function getElementOrThrow<T extends HTMLElement>(\n  selector: string, node: ParentNode = document\n): T {\n  const el = getElement<T>(selector, node)\n  if (typeof el === \"undefined\")\n    throw new ReferenceError(\n      `Missing element: expected \"${selector}\" to be present`\n    )\n  return el\n}\n\n/**\n * Retrieve the currently active element\n *\n * @returns Element or nothing\n */\nexport function getActiveElement(): HTMLElement | undefined {\n  return document.activeElement instanceof HTMLElement\n    ? document.activeElement\n    : undefined\n}\n\n/**\n * Retrieve all elements matching the query selector\n *\n * @template T - Element type\n *\n * @param selector - Query selector\n * @param node - Node of reference\n *\n * @returns Elements\n */\nexport function getElements<T extends keyof HTMLElementTagNameMap>(\n  selector: T, node?: ParentNode\n): HTMLElementTagNameMap[T][]\n\nexport function getElements<T extends HTMLElement>(\n  selector: string, node?: ParentNode\n): T[]\n\nexport function getElements<T extends HTMLElement>(\n  selector: string, node: ParentNode = document\n): T[] {\n  return Array.from(node.querySelectorAll<T>(selector))\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Create an element\n *\n * @template T - Tag name type\n *\n * @param tagName - Tag name\n *\n * @returns Element\n */\nexport function createElement<T extends keyof HTMLElementTagNameMap>(\n  tagName: T\n): HTMLElementTagNameMap[T] {\n  return document.createElement(tagName)\n}\n\n/**\n * Replace an element with the given list of nodes\n *\n * @param el - Element\n * @param nodes - Replacement nodes\n */\nexport function replaceElement(\n  el: HTMLElement, ...nodes: Node[]\n): void {\n  el.replaceWith(...nodes)\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Observable, fromEvent, merge } from \"rxjs\"\nimport { map, startWith } from \"rxjs/operators\"\n\nimport { getActiveElement } from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set element focus\n *\n * @param el - Element\n * @param value - Whether the element should be focused\n */\nexport function setElementFocus(\n  el: HTMLElement, value = true\n): void {\n  if (value)\n    el.focus()\n  else\n    el.blur()\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Watch element focus\n *\n * @param el - Element\n *\n * @returns Element focus observable\n */\nexport function watchElementFocus(\n  el: HTMLElement\n): Observable<boolean> {\n  return merge(\n    fromEvent<FocusEvent>(el, \"focus\"),\n    fromEvent<FocusEvent>(el, \"blur\")\n  )\n    .pipe(\n      map(({ type }) => type === \"focus\"),\n      startWith(el === getActiveElement())\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  NEVER,\n  Observable,\n  Subject,\n  defer,\n  of\n} from \"rxjs\"\nimport {\n  filter,\n  finalize,\n  map,\n  shareReplay,\n  startWith,\n  switchMap,\n  tap\n} from \"rxjs/operators\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Element offset\n */\nexport interface ElementSize {\n  width: number                        /* Element width */\n  height: number                       /* Element height */\n}\n\n/* ----------------------------------------------------------------------------\n * Data\n * ------------------------------------------------------------------------- */\n\n/**\n * Resize observer entry subject\n */\nconst entry$ = new Subject<ResizeObserverEntry>()\n\n/**\n * Resize observer observable\n *\n * This observable will create a `ResizeObserver` on the first subscription\n * and will automatically terminate it when there are no more subscribers.\n * It's quite important to centralize observation in a single `ResizeObserver`,\n * as the performance difference can be quite dramatic, as the link shows.\n *\n * @see https://bit.ly/3iIYfEm - Google Groups on performance\n */\nconst observer$ = defer(() => of(\n  new ResizeObserver(entries => {\n    for (const entry of entries)\n      entry$.next(entry)\n  })\n))\n  .pipe(\n    switchMap(resize => NEVER.pipe(startWith(resize))\n      .pipe(\n        finalize(() => resize.disconnect())\n      )\n    ),\n    shareReplay(1)\n  )\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve element size\n *\n * @param el - Element\n *\n * @returns Element size\n */\nexport function getElementSize(el: HTMLElement): ElementSize {\n  return {\n    width:  el.offsetWidth,\n    height: el.offsetHeight\n  }\n}\n\n/**\n * Retrieve element content size, i.e. including overflowing content\n *\n * @param el - Element\n *\n * @returns Element size\n */\nexport function getElementContentSize(el: HTMLElement): ElementSize {\n  return {\n    width:  el.scrollWidth,\n    height: el.scrollHeight\n  }\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Watch element size\n *\n * This function returns an observable that subscribes to a single internal\n * instance of `ResizeObserver` upon subscription, and emit resize events until\n * termination. Note that this function should not be called with the same\n * element twice, as the first unsubscription will terminate observation.\n *\n * Sadly, we can't use the `DOMRect` objects returned by the observer, because\n * we need the emitted values to be consistent with `getElementSize`, which will\n * return the used values (rounded) and not actual values (unrounded). Thus, we\n * use the `offset*` properties. See the linked GitHub issue.\n *\n * @see https://bit.ly/3m0k3he - GitHub issue\n *\n * @param el - Element\n *\n * @returns Element size observable\n */\nexport function watchElementSize(\n  el: HTMLElement\n): Observable<ElementSize> {\n  return observer$\n    .pipe(\n      tap(observer => observer.observe(el)),\n      switchMap(observer => entry$\n        .pipe(\n          filter(({ target }) => target === el),\n          finalize(() => observer.unobserve(el)),\n          map(() => getElementSize(el))\n        )\n      ),\n      startWith(getElementSize(el))\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Observable, fromEvent, merge } from \"rxjs\"\nimport {\n  distinctUntilChanged,\n  map,\n  startWith\n} from \"rxjs/operators\"\n\nimport {\n  getElementContentSize,\n  getElementSize\n} from \"../size\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Element offset\n */\nexport interface ElementOffset {\n  x: number                            /* Horizontal offset */\n  y: number                            /* Vertical offset */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve element offset\n *\n * @param el - Element\n *\n * @returns Element offset\n */\nexport function getElementOffset(el: HTMLElement): ElementOffset {\n  return {\n    x: el.scrollLeft,\n    y: el.scrollTop\n  }\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Watch element offset\n *\n * @param el - Element\n *\n * @returns Element offset observable\n */\nexport function watchElementOffset(\n  el: HTMLElement\n): Observable<ElementOffset> {\n  return merge(\n    fromEvent(el, \"scroll\"),\n    fromEvent(window, \"resize\")\n  )\n    .pipe(\n      map(() => getElementOffset(el)),\n      startWith(getElementOffset(el))\n    )\n}\n\n/**\n * Watch element threshold\n *\n * This function returns an observable which emits whether the bottom scroll\n * offset of an elements is within a certain threshold.\n *\n * @param el - Element\n * @param threshold - Threshold\n *\n * @returns Element threshold observable\n */\nexport function watchElementThreshold(\n  el: HTMLElement, threshold = 16\n): Observable<boolean> {\n  return watchElementOffset(el)\n    .pipe(\n      map(({ y }) => {\n        const visible = getElementSize(el)\n        const content = getElementContentSize(el)\n        return y >= (\n          content.height - visible.height - threshold\n        )\n      }),\n      distinctUntilChanged()\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set element text selection\n *\n * @param el - Element\n */\nexport function setElementSelection(\n  el: HTMLElement\n): void {\n  if (el instanceof HTMLInputElement)\n    el.select()\n  else\n    throw new Error(\"Not implemented\")\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Observable, fromEvent } from \"rxjs\"\nimport { map, startWith } from \"rxjs/operators\"\n\nimport { getElementOrThrow } from \"../element\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Toggle\n */\nexport type Toggle =\n  | \"drawer\"                           /* Toggle for drawer */\n  | \"search\"                           /* Toggle for search */\n\n/* ----------------------------------------------------------------------------\n * Data\n * ------------------------------------------------------------------------- */\n\n/**\n * Toggle map\n */\nconst toggles: Record<Toggle, HTMLInputElement> = {\n  drawer: getElementOrThrow(\"[data-md-toggle=drawer]\"),\n  search: getElementOrThrow(\"[data-md-toggle=search]\")\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve the value of a toggle\n *\n * @param name - Toggle\n *\n * @returns Toggle value\n */\nexport function getToggle(name: Toggle): boolean {\n  return toggles[name].checked\n}\n\n/**\n * Set toggle\n *\n * Simulating a click event seems to be the most cross-browser compatible way\n * of changing the value while also emitting a `change` event. Before, Material\n * used `CustomEvent` to programmatically change the value of a toggle, but this\n * is a much simpler and cleaner solution which doesn't require a polyfill.\n *\n * @param name - Toggle\n * @param value - Toggle value\n */\nexport function setToggle(name: Toggle, value: boolean): void {\n  if (toggles[name].checked !== value)\n    toggles[name].click()\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Watch toggle\n *\n * @param name - Toggle\n *\n * @returns Toggle value observable\n */\nexport function watchToggle(name: Toggle): Observable<boolean> {\n  const el = toggles[name]\n  return fromEvent(el, \"change\")\n    .pipe(\n      map(() => el.checked),\n      startWith(el.checked)\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Observable, fromEvent } from \"rxjs\"\nimport { filter, map, share } from \"rxjs/operators\"\n\nimport { getActiveElement } from \"../element\"\nimport { getToggle } from \"../toggle\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Keyboard mode\n */\nexport type KeyboardMode =\n  | \"global\"                           /* Global */\n  | \"search\"                           /* Search is open */\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Keyboard\n */\nexport interface Keyboard {\n  mode: KeyboardMode                   /* Keyboard mode */\n  type: string                         /* Key type */\n  claim(): void                        /* Key claim */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Check whether an element may receive keyboard input\n *\n * @param el - Element\n *\n * @returns Test result\n */\nfunction isSusceptibleToKeyboard(el: HTMLElement): boolean {\n  switch (el.tagName) {\n\n    /* Form elements */\n    case \"INPUT\":\n    case \"SELECT\":\n    case \"TEXTAREA\":\n      return true\n\n    /* Everything else */\n    default:\n      return el.isContentEditable\n  }\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch keyboard\n *\n * @returns Keyboard observable\n */\nexport function watchKeyboard(): Observable<Keyboard> {\n  return fromEvent<KeyboardEvent>(window, \"keydown\")\n    .pipe(\n      filter(ev => !(ev.metaKey || ev.ctrlKey)),\n      map(ev => ({\n        mode: getToggle(\"search\") ? \"search\" : \"global\",\n        type: ev.key,\n        claim() {\n          ev.preventDefault()\n          ev.stopPropagation()\n        }\n      } as Keyboard)),\n      filter(({ mode }) => {\n        if (mode === \"global\") {\n          const active = getActiveElement()\n          if (typeof active !== \"undefined\")\n            return !isSusceptibleToKeyboard(active)\n        }\n        return true\n      }),\n      share()\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Subject } from \"rxjs\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve location\n *\n * This function returns a `URL` object (and not `Location`) to normalize the\n * typings across the application. Furthermore, locations need to be tracked\n * without setting them and `Location` is a singleton which represents the\n * current location.\n *\n * @returns URL\n */\nexport function getLocation(): URL {\n  return new URL(location.href)\n}\n\n/**\n * Set location\n *\n * @param url - URL to change to\n */\nexport function setLocation(url: URL): void {\n  location.href = url.href\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Watch location\n *\n * @returns Location subject\n */\nexport function watchLocation(): Subject<URL> {\n  return new Subject<URL>()\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Observable, fromEvent, of } from \"rxjs\"\nimport { filter, map, share, startWith, switchMap } from \"rxjs/operators\"\n\nimport { createElement, getElement } from \"~/browser\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve location hash\n *\n * @returns Location hash\n */\nexport function getLocationHash(): string {\n  return location.hash.substring(1)\n}\n\n/**\n * Set location hash\n *\n * Setting a new fragment identifier via `location.hash` will have no effect\n * if the value doesn't change. When a new fragment identifier is set, we want\n * the browser to target the respective element at all times, which is why we\n * use this dirty little trick.\n *\n * @param hash - Location hash\n */\nexport function setLocationHash(hash: string): void {\n  const el = createElement(\"a\")\n  el.href = hash\n  el.addEventListener(\"click\", ev => ev.stopPropagation())\n  el.click()\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Watch location hash\n *\n * @returns Location hash observable\n */\nexport function watchLocationHash(): Observable<string> {\n  return fromEvent<HashChangeEvent>(window, \"hashchange\")\n    .pipe(\n      map(getLocationHash),\n      startWith(getLocationHash()),\n      filter(hash => hash.length > 0),\n      share()\n    )\n}\n\n/**\n * Watch location target\n *\n * @returns Location target observable\n */\nexport function watchLocationTarget(): Observable<HTMLElement> {\n  return watchLocationHash()\n    .pipe(\n      switchMap(id => of(getElement(`[id=\"${id}\"]`)!))\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  NEVER,\n  Observable,\n  fromEvent,\n  fromEventPattern\n} from \"rxjs\"\nimport {\n  mapTo,\n  startWith,\n  switchMap\n} from \"rxjs/operators\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch media query\n *\n * Note that although `MediaQueryList.addListener` is deprecated we have to\n * use it, because it's the only way to ensure proper downward compatibility.\n *\n * @see https://bit.ly/3dUBH2m - GitHub issue\n *\n * @param query - Media query\n *\n * @returns Media observable\n */\nexport function watchMedia(query: string): Observable<boolean> {\n  const media = matchMedia(query)\n  return fromEventPattern<boolean>(next => (\n    media.addListener(() => next(media.matches))\n  ))\n    .pipe(\n      startWith(media.matches)\n    )\n}\n\n/**\n * Watch print mode, cross-browser\n *\n * @returns Print mode observable\n */\nexport function watchPrint(): Observable<void> {\n  return fromEvent(window, \"beforeprint\")\n    .pipe(\n      mapTo(undefined)\n    )\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Toggle an observable with a media observable\n *\n * @template T - Data type\n *\n * @param query$ - Media observable\n * @param factory - Observable factory\n *\n * @returns Toggled observable\n */\nexport function at<T>(\n  query$: Observable<boolean>, factory: () => Observable<T>\n): Observable<T> {\n  return query$\n    .pipe(\n      switchMap(active => active ? factory() : NEVER)\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Observable, from } from \"rxjs\"\nimport {\n  filter,\n  map,\n  shareReplay,\n  switchMap\n} from \"rxjs/operators\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Fetch the given URL\n *\n * @param url - Request URL\n * @param options - Options\n *\n * @returns Response observable\n */\nexport function request(\n  url: URL | string, options: RequestInit = { credentials: \"same-origin\" }\n): Observable<Response> {\n  return from(fetch(`${url}`, options))\n    .pipe(\n      filter(res => res.status === 200),\n    )\n}\n\n/**\n * Fetch JSON from the given URL\n *\n * @template T - Data type\n *\n * @param url - Request URL\n * @param options - Options\n *\n * @returns Data observable\n */\nexport function requestJSON<T>(\n  url: URL | string, options?: RequestInit\n): Observable<T> {\n  return request(url, options)\n    .pipe(\n      switchMap(res => res.json()),\n      shareReplay(1)\n    )\n}\n\n/**\n * Fetch XML from the given URL\n *\n * @param url - Request URL\n * @param options - Options\n *\n * @returns Data observable\n */\nexport function requestXML(\n  url: URL | string, options?: RequestInit\n): Observable<Document> {\n  const dom = new DOMParser()\n  return request(url, options)\n    .pipe(\n      switchMap(res => res.text()),\n      map(res => dom.parseFromString(res, \"text/xml\")),\n      shareReplay(1)\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Observable, fromEvent, merge } from \"rxjs\"\nimport { map, startWith } from \"rxjs/operators\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Viewport offset\n */\nexport interface ViewportOffset {\n  x: number                            /* Horizontal offset */\n  y: number                            /* Vertical offset */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve viewport offset\n *\n * On iOS Safari, viewport offset can be negative due to overflow scrolling.\n * As this may induce strange behaviors downstream, we'll just limit it to 0.\n *\n * @returns Viewport offset\n */\nexport function getViewportOffset(): ViewportOffset {\n  return {\n    x: Math.max(0, pageXOffset),\n    y: Math.max(0, pageYOffset)\n  }\n}\n\n/**\n * Set viewport offset\n *\n * @param offset - Viewport offset\n */\nexport function setViewportOffset(\n  { x, y }: Partial<ViewportOffset>\n): void {\n  window.scrollTo(x || 0, y || 0)\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Watch viewport offset\n *\n * @returns Viewport offset observable\n */\nexport function watchViewportOffset(): Observable<ViewportOffset> {\n  return merge(\n    fromEvent(window, \"scroll\", { passive: true }),\n    fromEvent(window, \"resize\", { passive: true })\n  )\n    .pipe(\n      map(getViewportOffset),\n      startWith(getViewportOffset())\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Observable, fromEvent } from \"rxjs\"\nimport { map, startWith } from \"rxjs/operators\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Viewport size\n */\nexport interface ViewportSize {\n  width: number                        /* Viewport width */\n  height: number                       /* Viewport height */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve viewport size\n *\n * @returns Viewport size\n */\nexport function getViewportSize(): ViewportSize {\n  return {\n    width:  innerWidth,\n    height: innerHeight\n  }\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Watch viewport size\n *\n * @returns Viewport size observable\n */\nexport function watchViewportSize(): Observable<ViewportSize> {\n  return fromEvent(window, \"resize\", { passive: true })\n    .pipe(\n      map(getViewportSize),\n      startWith(getViewportSize())\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Observable, combineLatest } from \"rxjs\"\nimport {\n  distinctUntilKeyChanged,\n  map,\n  shareReplay\n} from \"rxjs/operators\"\n\nimport { Header } from \"~/components\"\n\nimport {\n  ViewportOffset,\n  watchViewportOffset\n} from \"../offset\"\nimport {\n  ViewportSize,\n  watchViewportSize\n} from \"../size\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Viewport\n */\nexport interface Viewport {\n  offset: ViewportOffset               /* Viewport offset */\n  size: ViewportSize                   /* Viewport size */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch at options\n */\ninterface WatchAtOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch viewport\n *\n * @returns Viewport observable\n */\nexport function watchViewport(): Observable<Viewport> {\n  return combineLatest([\n    watchViewportOffset(),\n    watchViewportSize()\n  ])\n    .pipe(\n      map(([offset, size]) => ({ offset, size })),\n      shareReplay(1)\n    )\n}\n\n/**\n * Watch viewport relative to element\n *\n * @param el - Element\n * @param options - Options\n *\n * @returns Viewport observable\n */\nexport function watchViewportAt(\n  el: HTMLElement, { viewport$, header$ }: WatchAtOptions\n): Observable<Viewport> {\n  const size$ = viewport$\n    .pipe(\n      distinctUntilKeyChanged(\"size\")\n    )\n\n  /* Compute element offset */\n  const offset$ = combineLatest([size$, header$])\n    .pipe(\n      map((): ViewportOffset => ({\n        x: el.offsetLeft,\n        y: el.offsetTop\n      }))\n    )\n\n  /* Compute relative viewport, return hot observable */\n  return combineLatest([header$, viewport$, offset$])\n    .pipe(\n      map(([{ height }, { offset, size }, { x, y }]) => ({\n        offset: {\n          x: offset.x - x,\n          y: offset.y - y + height\n        },\n        size\n      }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Observable, Subject, fromEvent } from \"rxjs\"\nimport {\n  map,\n  share,\n  switchMapTo,\n  tap,\n  throttle\n} from \"rxjs/operators\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Worker message\n */\nexport interface WorkerMessage {\n  type: unknown                        /* Message type */\n  data?: unknown                       /* Message data */\n}\n\n/**\n * Worker handler\n *\n * @template T - Message type\n */\nexport interface WorkerHandler<\n  T extends WorkerMessage\n> {\n  tx$: Subject<T>                      /* Message transmission subject */\n  rx$: Observable<T>                   /* Message receive observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n *\n * @template T - Worker message type\n */\ninterface WatchOptions<T extends WorkerMessage> {\n  tx$: Observable<T>                   /* Message transmission observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch a web worker\n *\n * This function returns an observable that sends all values emitted by the\n * message observable to the web worker. Web worker communication is expected\n * to be bidirectional (request-response) and synchronous. Messages that are\n * emitted during a pending request are throttled, the last one is emitted.\n *\n * @param worker - Web worker\n * @param options - Options\n *\n * @returns Worker message observable\n */\nexport function watchWorker<T extends WorkerMessage>(\n  worker: Worker, { tx$ }: WatchOptions<T>\n): Observable<T> {\n\n  /* Intercept messages from worker-like objects */\n  const rx$ = fromEvent<MessageEvent>(worker, \"message\")\n    .pipe(\n      map(({ data }) => data as T)\n    )\n\n  /* Send and receive messages, return hot observable */\n  return tx$\n    .pipe(\n      throttle(() => rx$, { leading: true, trailing: true }),\n      tap(message => worker.postMessage(message)),\n      switchMapTo(rx$),\n      share()\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { getElementOrThrow, getLocation } from \"~/browser\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Feature flag\n */\nexport type Flag =\n  | \"header.autohide\"                  /* Hide header */\n  | \"navigation.expand\"                /* Automatic expansion */\n  | \"navigation.instant\"               /* Instant loading */\n  | \"navigation.sections\"              /* Sections navigation */\n  | \"navigation.tabs\"                  /* Tabs navigation */\n  | \"navigation.top\"                   /* Back-to-top button */\n  | \"search.highlight\"                 /* Search highlighting */\n  | \"search.share\"                     /* Search sharing */\n  | \"search.suggest\"                   /* Search suggestions */\n  | \"toc.integrate\"                    /* Integrated table of contents */\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Translation\n */\nexport type Translation =\n  | \"clipboard.copy\"                   /* Copy to clipboard */\n  | \"clipboard.copied\"                 /* Copied to clipboard */\n  | \"search.config.lang\"               /* Search language */\n  | \"search.config.pipeline\"           /* Search pipeline */\n  | \"search.config.separator\"          /* Search separator */\n  | \"search.placeholder\"               /* Search */\n  | \"search.result.placeholder\"        /* Type to start searching */\n  | \"search.result.none\"               /* No matching documents */\n  | \"search.result.one\"                /* 1 matching document */\n  | \"search.result.other\"              /* # matching documents */\n  | \"search.result.more.one\"           /* 1 more on this page */\n  | \"search.result.more.other\"         /* # more on this page */\n  | \"search.result.term.missing\"       /* Missing */\n  | \"select.version.title\"             /* Version selector */\n\n/**\n * Translations\n */\nexport type Translations = Record<Translation, string>\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Versioning\n */\nexport interface Versioning {\n  provider: \"mike\"                     /* Version provider */\n}\n\n/**\n * Configuration\n */\nexport interface Config {\n  base: string                         /* Base URL */\n  features: Flag[]                     /* Feature flags */\n  translations: Translations           /* Translations */\n  search: string                       /* Search worker URL */\n  version?: Versioning                 /* Versioning */\n}\n\n/* ----------------------------------------------------------------------------\n * Data\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve global configuration and make base URL absolute\n */\nconst script = getElementOrThrow(\"#__config\")\nconst config: Config = JSON.parse(script.textContent!)\nconfig.base = `${new URL(config.base, getLocation())}`\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve global configuration\n *\n * @returns Global configuration\n */\nexport function configuration(): Config {\n  return config\n}\n\n/**\n * Check whether a feature flag is enabled\n *\n * @param flag - Feature flag\n *\n * @returns Test result\n */\nexport function feature(flag: Flag): boolean {\n  return config.features.includes(flag)\n}\n\n/**\n * Retrieve the translation for the given key\n *\n * @param key - Key to be translated\n * @param value - Positional value, if any\n *\n * @returns Translation\n */\nexport function translation(\n  key: Translation, value?: string | number\n): string {\n  return typeof value !== \"undefined\"\n    ? config.translations[key].replace(\"#\", value.toString())\n    : config.translations[key]\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { getElementOrThrow, getElements } from \"~/browser\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Component\n */\nexport type ComponentType =\n  | \"announce\"                         /* Announcement bar */\n  | \"container\"                        /* Container */\n  | \"content\"                          /* Content */\n  | \"dialog\"                           /* Dialog */\n  | \"header\"                           /* Header */\n  | \"header-title\"                     /* Header title */\n  | \"header-topic\"                     /* Header topic */\n  | \"main\"                             /* Main area */\n  | \"palette\"                          /* Color palette */\n  | \"search\"                           /* Search */\n  | \"search-query\"                     /* Search input */\n  | \"search-result\"                    /* Search results */\n  | \"search-share\"                     /* Search sharing */\n  | \"search-suggest\"                   /* Search suggestions */\n  | \"sidebar\"                          /* Sidebar */\n  | \"skip\"                             /* Skip link */\n  | \"source\"                           /* Repository information */\n  | \"tabs\"                             /* Navigation tabs */\n  | \"toc\"                              /* Table of contents */\n  | \"top\"                              /* Back-to-top button */\n\n/**\n * A component\n *\n * @template T - Component type\n * @template U - Reference type\n */\nexport type Component<\n  T extends {} = {},\n  U extends HTMLElement = HTMLElement\n> =\n  T & {\n    ref: U                             /* Component reference */\n  }\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Component type map\n */\ninterface ComponentTypeMap {\n  \"announce\": HTMLElement              /* Announcement bar */\n  \"container\": HTMLElement             /* Container */\n  \"content\": HTMLElement               /* Content */\n  \"dialog\": HTMLElement                /* Dialog */\n  \"header\": HTMLElement                /* Header */\n  \"header-title\": HTMLElement          /* Header title */\n  \"header-topic\": HTMLElement          /* Header topic */\n  \"main\": HTMLElement                  /* Main area */\n  \"palette\": HTMLElement               /* Color palette */\n  \"search\": HTMLElement                /* Search */\n  \"search-query\": HTMLInputElement     /* Search input */\n  \"search-result\": HTMLElement         /* Search results */\n  \"search-share\": HTMLAnchorElement    /* Search sharing */\n  \"search-suggest\": HTMLElement        /* Search suggestions */\n  \"sidebar\": HTMLElement               /* Sidebar */\n  \"skip\": HTMLAnchorElement            /* Skip link */\n  \"source\": HTMLAnchorElement          /* Repository information */\n  \"tabs\": HTMLElement                  /* Navigation tabs */\n  \"toc\": HTMLElement                   /* Table of contents */\n  \"top\": HTMLAnchorElement             /* Back-to-top button */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve the element for a given component or throw a reference error\n *\n * @template T - Component type\n *\n * @param type - Component type\n * @param node - Node of reference\n *\n * @returns Element\n */\nexport function getComponentElement<T extends ComponentType>(\n  type: T, node: ParentNode = document\n): ComponentTypeMap[T] {\n  return getElementOrThrow(`[data-md-component=${type}]`, node)\n}\n\n/**\n * Retrieve all elements for a given component\n *\n * @template T - Component type\n *\n * @param type - Component type\n * @param node - Node of reference\n *\n * @returns Elements\n */\nexport function getComponentElements<T extends ComponentType>(\n  type: T, node: ParentNode = document\n): ComponentTypeMap[T][] {\n  return getElements(`[data-md-component=${type}]`, node)\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport ClipboardJS from \"clipboard\"\nimport {\n  NEVER,\n  Observable,\n  Subject,\n  fromEvent,\n  merge,\n  of\n} from \"rxjs\"\nimport {\n  distinctUntilKeyChanged,\n  map,\n  switchMap,\n  tap,\n  withLatestFrom\n} from \"rxjs/operators\"\n\nimport { resetFocusable, setFocusable } from \"~/actions\"\nimport {\n  Viewport,\n  getElementContentSize,\n  getElementSize,\n  getElements,\n  watchMedia\n} from \"~/browser\"\nimport { renderClipboardButton } from \"~/templates\"\n\nimport { Component } from \"../../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Code block\n */\nexport interface CodeBlock {\n  scroll: boolean                      /* Code block overflows */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Data\n * ------------------------------------------------------------------------- */\n\n/**\n * Global index for Clipboard.js integration\n */\nlet index = 0\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch code block\n *\n * This function monitors size changes of the viewport, as well as switches of\n * content tabs with embedded code blocks, as both may trigger overflow.\n *\n * @param el - Code block element\n * @param options - Options\n *\n * @returns Code block observable\n */\nexport function watchCodeBlock(\n  el: HTMLElement, { viewport$ }: WatchOptions\n): Observable<CodeBlock> {\n  const container$ = of(el)\n    .pipe(\n      switchMap(child => {\n        const container = child.closest(\"[data-tabs]\")\n        if (container instanceof HTMLElement) {\n          return merge(\n            ...getElements(\"input\", container)\n              .map(input => fromEvent(input, \"change\"))\n          )\n        }\n        return NEVER\n      })\n    )\n\n  /* Check overflow on resize and tab change */\n  return merge(\n    viewport$.pipe(distinctUntilKeyChanged(\"size\")),\n    container$\n  )\n    .pipe(\n      map(() => {\n        const visible = getElementSize(el)\n        const content = getElementContentSize(el)\n        return {\n          scroll: content.width > visible.width\n        }\n      }),\n      distinctUntilKeyChanged(\"scroll\")\n    )\n}\n\n/**\n * Mount code block\n *\n * This function ensures that an overflowing code block is focusable through\n * keyboard, so it can be scrolled without a mouse to improve on accessibility.\n *\n * @param el - Code block element\n * @param options - Options\n *\n * @returns Code block component observable\n */\nexport function mountCodeBlock(\n  el: HTMLElement, options: MountOptions\n): Observable<Component<CodeBlock>> {\n  const internal$ = new Subject<CodeBlock>()\n  internal$\n    .pipe(\n      withLatestFrom(watchMedia(\"(hover)\"))\n    )\n      .subscribe(([{ scroll }, hover]) => {\n        if (scroll && hover)\n          setFocusable(el)\n        else\n          resetFocusable(el)\n      })\n\n  /* Render button for Clipboard.js integration */\n  if (ClipboardJS.isSupported()) {\n    const parent = el.closest(\"pre\")!\n    parent.id = `__code_${index++}`\n    parent.insertBefore(\n      renderClipboardButton(parent.id),\n      el\n    )\n  }\n\n  /* Create and return component */\n  return watchCodeBlock(el, options)\n    .pipe(\n      tap(internal$),\n      map(state => ({ ref: el, ...state }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set focusable property\n *\n * @param el - Element\n * @param value - Tabindex value\n */\nexport function setFocusable(\n  el: HTMLElement, value = 0\n): void {\n  el.setAttribute(\"tabindex\", value.toString())\n}\n\n/**\n * Reset focusable property\n *\n * @param el - Element\n */\nexport function resetFocusable(\n  el: HTMLElement\n): void {\n  el.removeAttribute(\"tabindex\")\n}\n\n/**\n * Set scroll lock\n *\n * @param el - Scrollable element\n * @param value - Vertical offset\n */\nexport function setScrollLock(\n  el: HTMLElement, value: number\n): void {\n  el.setAttribute(\"data-md-state\", \"lock\")\n  el.style.top = `-${value}px`\n}\n\n/**\n * Reset scroll lock\n *\n * @param el - Scrollable element\n */\nexport function resetScrollLock(\n  el: HTMLElement\n): void {\n  const value = -1 * parseInt(el.style.top, 10)\n  el.removeAttribute(\"data-md-state\")\n  el.style.top = \"\"\n  if (value)\n    window.scrollTo(0, value)\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set anchor state\n *\n * @param el - Anchor element\n * @param state - Anchor state\n */\nexport function setAnchorState(\n  el: HTMLElement, state: \"blur\"\n): void {\n  el.setAttribute(\"data-md-state\", state)\n}\n\n/**\n * Reset anchor state\n *\n * @param el - Anchor element\n */\nexport function resetAnchorState(\n  el: HTMLElement\n): void {\n  el.removeAttribute(\"data-md-state\")\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Set anchor active\n *\n * @param el - Anchor element\n * @param value - Whether the anchor is active\n */\nexport function setAnchorActive(\n  el: HTMLElement, value: boolean\n): void {\n  el.classList.toggle(\"md-nav__link--active\", value)\n}\n\n/**\n * Reset anchor active\n *\n * @param el - Anchor element\n */\nexport function resetAnchorActive(\n  el: HTMLElement\n): void {\n  el.classList.remove(\"md-nav__link--active\")\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set dialog message\n *\n * @param el - Dialog element\n * @param value - Dialog message\n */\nexport function setDialogMessage(\n  el: HTMLElement, value: string\n): void {\n  el.firstElementChild!.innerHTML = value\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Set dialog state\n *\n * @param el - Dialog element\n * @param state - Dialog state\n */\nexport function setDialogState(\n  el: HTMLElement, state: \"open\"\n): void {\n  el.setAttribute(\"data-md-state\", state)\n}\n\n/**\n * Reset dialog state\n *\n * @param el - Dialog element\n */\nexport function resetDialogState(\n  el: HTMLElement\n): void {\n  el.removeAttribute(\"data-md-state\")\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set header state\n *\n * @param el - Header element\n * @param state - Header state\n */\nexport function setHeaderState(\n  el: HTMLElement, state: \"shadow\" | \"hidden\"\n): void {\n  el.setAttribute(\"data-md-state\", state)\n}\n\n/**\n * Reset header state\n *\n * @param el - Header element\n */\nexport function resetHeaderState(\n  el: HTMLElement\n): void {\n  el.removeAttribute(\"data-md-state\")\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set header title state\n *\n * @param el - Header title element\n * @param state - Header title state\n */\nexport function setHeaderTitleState(\n  el: HTMLElement, state: \"active\"\n): void {\n  el.setAttribute(\"data-md-state\", state)\n}\n\n/**\n * Reset header title state\n *\n * @param el - Header title element\n */\nexport function resetHeaderTitleState(\n  el: HTMLElement\n): void {\n  el.removeAttribute(\"data-md-state\")\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { translation } from \"~/_\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set search query placeholder\n *\n * @param el - Search query element\n * @param value - Placeholder\n */\nexport function setSearchQueryPlaceholder(\n  el: HTMLInputElement, value: string\n): void {\n  el.placeholder = value\n}\n\n/**\n * Reset search query placeholder\n *\n * @param el - Search query element\n */\nexport function resetSearchQueryPlaceholder(\n  el: HTMLInputElement\n): void {\n  el.placeholder = translation(\"search.placeholder\")\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { JSX as JSXInternal } from \"preact\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * HTML attributes\n */\ntype Attributes =\n  & JSXInternal.HTMLAttributes\n  & JSXInternal.SVGAttributes\n  & Record<string, any>\n\n/**\n * Child element\n */\ntype Child =\n  | HTMLElement\n  | Text\n  | string\n  | number\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Append a child node to an element\n *\n * @param el - Element\n * @param child - Child node(s)\n */\nfunction appendChild(el: HTMLElement, child: Child | Child[]): void {\n\n  /* Handle primitive types (including raw HTML) */\n  if (typeof child === \"string\" || typeof child === \"number\") {\n    el.innerHTML += child.toString()\n\n  /* Handle nodes */\n  } else if (child instanceof Node) {\n    el.appendChild(child)\n\n  /* Handle nested children */\n  } else if (Array.isArray(child)) {\n    for (const node of child)\n      appendChild(el, node)\n  }\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * JSX factory\n *\n * @param tag - HTML tag\n * @param attributes - HTML attributes\n * @param children - Child elements\n *\n * @returns Element\n */\nexport function h(\n  tag: string, attributes: Attributes | null, ...children: Child[]\n): HTMLElement {\n  const el = document.createElement(tag)\n\n  /* Set attributes, if any */\n  if (attributes)\n    for (const attr of Object.keys(attributes))\n      if (typeof attributes[attr] !== \"boolean\")\n        el.setAttribute(attr, attributes[attr])\n      else if (attributes[attr])\n        el.setAttribute(attr, \"\")\n\n  /* Append child nodes */\n  for (const child of children)\n    appendChild(el, child)\n\n  /* Return element */\n  return el\n}\n\n/* ----------------------------------------------------------------------------\n * Namespace\n * ------------------------------------------------------------------------- */\n\nexport declare namespace h {\n  namespace JSX {\n    type Element = HTMLElement\n    type IntrinsicElements = JSXInternal.IntrinsicElements\n  }\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Truncate a string after the given number of characters\n *\n * This is not a very reasonable approach, since the summaries kind of suck.\n * It would be better to create something more intelligent, highlighting the\n * search occurrences and making a better summary out of it, but this note was\n * written three years ago, so who knows if we'll ever fix it.\n *\n * @param value - Value to be truncated\n * @param n - Number of characters\n *\n * @returns Truncated value\n */\nexport function truncate(value: string, n: number): string {\n  let i = n\n  if (value.length > i) {\n    while (value[i] !== \" \" && --i > 0) { /* keep eating */ }\n    return `${value.substring(0, i)}...`\n  }\n  return value\n}\n\n/**\n * Round a number for display with repository facts\n *\n * This is a reverse-engineered version of GitHub's weird rounding algorithm\n * for stars, forks and all other numbers. While all numbers below `1,000` are\n * returned as-is, bigger numbers are converted to fixed numbers:\n *\n * - `1,049` => `1k`\n * - `1,050` => `1.1k`\n * - `1,949` => `1.9k`\n * - `1,950` => `2k`\n *\n * @param value - Original value\n *\n * @returns Rounded value\n */\nexport function round(value: number): string {\n  if (value > 999) {\n    const digits = +((value - 950) % 1000 > 99)\n    return `${((value + 0.000001) / 1000).toFixed(digits)}k`\n  } else {\n    return value.toString()\n  }\n}\n\n/**\n * Simple hash function\n *\n * @see https://bit.ly/2wsVjJ4 - Original source\n *\n * @param value - Value to be hashed\n *\n * @returns Hash as 32bit integer\n */\nexport function hash(value: string): number {\n  let h = 0\n  for (let i = 0, len = value.length; i < len; i++) {\n    h  = ((h << 5) - h) + value.charCodeAt(i)\n    h |= 0 // Convert to 32bit integer\n  }\n  return h\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { translation } from \"~/_\"\nimport { round } from \"~/utilities\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set number of search results\n *\n * @param el - Search result metadata element\n * @param value - Number of results\n */\nexport function setSearchResultMeta(\n  el: HTMLElement, value: number\n): void {\n  switch (value) {\n\n    /* No results */\n    case 0:\n      el.textContent = translation(\"search.result.none\")\n      break\n\n    /* One result */\n    case 1:\n      el.textContent = translation(\"search.result.one\")\n      break\n\n    /* Multiple result */\n    default:\n      el.textContent = translation(\"search.result.other\", round(value))\n  }\n}\n\n/**\n * Reset number of search results\n *\n * @param el - Search result metadata element\n */\nexport function resetSearchResultMeta(\n  el: HTMLElement\n): void {\n  el.textContent = translation(\"search.result.placeholder\")\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Add an element to the search result list\n *\n * @param el - Search result list element\n * @param child - Search result element\n */\nexport function addToSearchResultList(\n  el: HTMLElement, child: Element\n): void {\n  el.appendChild(child)\n}\n\n/**\n * Reset search result list\n *\n * @param el - Search result list element\n */\nexport function resetSearchResultList(\n  el: HTMLElement\n): void {\n  el.innerHTML = \"\"\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set sidebar offset\n *\n * @param el - Sidebar element\n * @param value - Sidebar offset\n */\nexport function setSidebarOffset(\n  el: HTMLElement, value: number\n): void {\n  el.style.top = `${value}px`\n}\n\n/**\n * Reset sidebar offset\n *\n * @param el - Sidebar element\n */\nexport function resetSidebarOffset(\n  el: HTMLElement\n): void {\n  el.style.top = \"\"\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Set sidebar height\n *\n * This function doesn't set the height of the actual sidebar, but of its first\n * child \u2013 the `.md-sidebar__scrollwrap` element in order to mitigiate jittery\n * sidebars when the footer is scrolled into view. At some point we switched\n * from `absolute` / `fixed` positioning to `sticky` positioning, significantly\n * reducing jitter in some browsers (respectively Firefox and Safari) when\n * scrolling from the top. However, top-aligned sticky positioning means that\n * the sidebar snaps to the bottom when the end of the container is reached.\n * This is what leads to the mentioned jitter, as the sidebar's height may be\n * updated too slowly.\n *\n * This behaviour can be mitigiated by setting the height of the sidebar to `0`\n * while preserving the padding, and the height on its first element.\n *\n * @param el - Sidebar element\n * @param value - Sidebar height\n */\nexport function setSidebarHeight(\n  el: HTMLElement, value: number\n): void {\n  const scrollwrap = el.firstElementChild as HTMLElement\n  scrollwrap.style.height = `${value - 2 * scrollwrap.offsetTop}px`\n}\n\n/**\n * Reset sidebar height\n *\n * @param el - Sidebar element\n */\nexport function resetSidebarHeight(\n  el: HTMLElement\n): void {\n  const scrollwrap = el.firstElementChild as HTMLElement\n  scrollwrap.style.height = \"\"\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set repository facts\n *\n * @param el - Repository element\n * @param child - Repository facts element\n */\nexport function setSourceFacts(\n  el: HTMLElement, child: Element\n): void {\n  el.lastElementChild!.appendChild(child)\n}\n\n/**\n * Set repository state\n *\n * @param el - Repository element\n * @param state - Repository state\n */\nexport function setSourceState(\n  el: HTMLElement, state: \"done\"\n): void {\n  el.lastElementChild!.setAttribute(\"data-md-state\", state)\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set tabs state\n *\n * @param el - Tabs element\n * @param state - Tabs state\n */\nexport function setTabsState(\n  el: HTMLElement, state: \"hidden\"\n): void {\n  el.setAttribute(\"data-md-state\", state)\n}\n\n/**\n * Reset tabs state\n *\n * @param el - Tabs element\n */\nexport function resetTabsState(\n  el: HTMLElement\n): void {\n  el.removeAttribute(\"data-md-state\")\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set back-to-top state\n *\n * @param el - Back-to-top element\n * @param state - Back-to-top state\n */\nexport function setBackToTopState(\n  el: HTMLElement, state: \"hidden\"\n): void {\n  el.setAttribute(\"data-md-state\", state)\n}\n\n/**\n * Reset back-to-top state\n *\n * @param el - Back-to-top element\n */\nexport function resetBackToTopState(\n  el: HTMLElement\n): void {\n  el.removeAttribute(\"data-md-state\")\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Set back-to-top offset\n *\n * @param el - Back-to-top element\n * @param value - Back-to-top offset\n */\nexport function setBackToTopOffset(\n  el: HTMLElement, value: number\n): void {\n  el.style.top = `${value}px`\n}\n\n/**\n * Reset back-to-top offset\n *\n * @param el - Back-to-top element\n */\nexport function resetBackToTopOffset(\n  el: HTMLElement\n): void {\n  el.style.top = \"\"\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { translation } from \"~/_\"\nimport { h } from \"~/utilities\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Render a 'copy-to-clipboard' button\n *\n * @param id - Unique identifier\n *\n * @returns Element\n */\nexport function renderClipboardButton(id: string): HTMLElement {\n  return (\n    <button\n      class=\"md-clipboard md-icon\"\n      title={translation(\"clipboard.copy\")}\n      data-clipboard-target={`#${id} > code`}\n    ></button>\n  )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { feature, translation } from \"~/_\"\nimport {\n  SearchDocument,\n  SearchMetadata,\n  SearchResultItem\n} from \"~/integrations/search\"\nimport { h, truncate } from \"~/utilities\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Render flag\n */\nconst enum Flag {\n  TEASER = 1,                          /* Render teaser */\n  PARENT = 2                           /* Render as parent */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper function\n * ------------------------------------------------------------------------- */\n\n/**\n * Render a search document\n *\n * @param document - Search document\n * @param flag - Render flags\n *\n * @returns Element\n */\nfunction renderSearchDocument(\n  document: SearchDocument & SearchMetadata, flag: Flag\n): HTMLElement {\n  const parent = flag & Flag.PARENT\n  const teaser = flag & Flag.TEASER\n\n  /* Render missing query terms */\n  const missing = Object.keys(document.terms)\n    .filter(key => !document.terms[key])\n    .map(key => [<del>{key}</del>, \" \"])\n    .flat()\n    .slice(0, -1)\n\n  /* Assemble query string for highlighting */\n  const url = new URL(document.location)\n  if (feature(\"search.highlight\"))\n    url.searchParams.set(\"h\", Object.entries(document.terms)\n      .filter(([, match]) => match)\n      .reduce((highlight, [value]) => `${highlight} ${value}`.trim(), \"\")\n    )\n\n  /* Render article or section, depending on flags */\n  return (\n    <a href={`${url}`} class=\"md-search-result__link\" tabIndex={-1}>\n      <article\n        class={[\"md-search-result__article\", ...parent\n          ? [\"md-search-result__article--document\"]\n          : []\n        ].join(\" \")}\n        data-md-score={document.score.toFixed(2)}\n      >\n        {parent > 0 && <div class=\"md-search-result__icon md-icon\"></div>}\n        <h1 class=\"md-search-result__title\">{document.title}</h1>\n        {teaser > 0 && document.text.length > 0 &&\n          <p class=\"md-search-result__teaser\">\n            {truncate(document.text, 320)}\n          </p>\n        }\n        {teaser > 0 && missing.length > 0 &&\n          <p class=\"md-search-result__terms\">\n            {translation(\"search.result.term.missing\")}: {...missing}\n          </p>\n        }\n      </article>\n    </a>\n  )\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Render a search result\n *\n * @param result - Search result\n *\n * @returns Element\n */\nexport function renderSearchResultItem(\n  result: SearchResultItem\n): HTMLElement {\n  const threshold = result[0].score\n  const docs = [...result]\n\n  /* Find and extract parent article */\n  const parent = docs.findIndex(doc => !doc.location.includes(\"#\"))\n  const [article] = docs.splice(parent, 1)\n\n  /* Determine last index above threshold */\n  let index = docs.findIndex(doc => doc.score < threshold)\n  if (index === -1)\n    index = docs.length\n\n  /* Partition sections */\n  const best = docs.slice(0, index)\n  const more = docs.slice(index)\n\n  /* Render children */\n  const children = [\n    renderSearchDocument(article, Flag.PARENT | +(!parent && index === 0)),\n    ...best.map(section => renderSearchDocument(section, Flag.TEASER)),\n    ...more.length ? [\n      <details class=\"md-search-result__more\">\n        <summary tabIndex={-1}>\n          {more.length > 0 && more.length === 1\n            ? translation(\"search.result.more.one\")\n            : translation(\"search.result.more.other\", more.length)\n          }\n        </summary>\n        {...more.map(section => renderSearchDocument(section, Flag.TEASER))}\n      </details>\n    ] : []\n  ]\n\n  /* Render search result */\n  return (\n    <li class=\"md-search-result__item\">\n      {children}\n    </li>\n  )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { SourceFacts } from \"~/components\"\nimport { h, round } from \"~/utilities\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Render repository facts\n *\n * @param facts - Repository facts\n *\n * @returns Element\n */\nexport function renderSourceFacts(facts: SourceFacts): HTMLElement {\n  return (\n    <ul class=\"md-source__facts\">\n      {Object.entries(facts).map(([key, value]) => (\n        <li class={`md-source__fact md-source__fact--${key}`}>\n          {typeof value === \"number\" ? round(value) : value}\n        </li>\n      ))}\n    </ul>\n  )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { h } from \"~/utilities\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Render a table inside a wrapper to improve scrolling on mobile\n *\n * @param table - Table element\n *\n * @returns Element\n */\nexport function renderTable(table: HTMLElement): HTMLElement {\n  return (\n    <div class=\"md-typeset__scrollwrap\">\n      <div class=\"md-typeset__table\">\n        {table}\n      </div>\n    </div>\n  )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { configuration, translation } from \"~/_\"\nimport { h } from \"~/utilities\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Version\n */\nexport interface Version {\n  version: string                      /* Version identifier */\n  title: string                        /* Version title */\n  aliases: string[]                    /* Version aliases */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Render a version\n *\n * @param version - Version\n *\n * @returns Element\n */\nfunction renderVersion(version: Version): HTMLElement {\n  const config = configuration()\n\n  /* Ensure trailing slash, see https://bit.ly/3rL5u3f */\n  const url = new URL(`../${version.version}/`, config.base)\n  return (\n    <li class=\"md-version__item\">\n      <a href={url.toString()} class=\"md-version__link\">\n        {version.title}\n      </a>\n    </li>\n  )\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Render a version selector\n *\n * @param versions - Versions\n *\n * @returns Element\n */\nexport function renderVersionSelector(versions: Version[]): HTMLElement {\n  const config = configuration()\n\n  /* Determine active version */\n  const [, current] = config.base.match(/([^/]+)\\/?$/)!\n  const active =\n    versions.find(({ version, aliases }) => (\n      version === current || aliases.includes(current)\n    )) || versions[0]\n\n  /* Render version selector */\n  return (\n    <div class=\"md-version\">\n      <button\n        class=\"md-version__current\"\n        aria-label={translation(\"select.version.title\")}\n      >\n        {active.title}\n      </button>\n      <ul class=\"md-version__list\">\n        {versions.map(renderVersion)}\n      </ul>\n    </div>\n  )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Observable, Subject } from \"rxjs\"\nimport {\n  filter,\n  map,\n  mapTo,\n  mergeWith,\n  tap\n} from \"rxjs/operators\"\n\nimport { Component } from \"../../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Details\n */\nexport interface Details {}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  target$: Observable<HTMLElement>     /* Location target observable */\n  print$: Observable<void>             /* Print mode observable */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  target$: Observable<HTMLElement>     /* Location target observable */\n  print$: Observable<void>             /* Print mode observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch details\n *\n * @param el - Details element\n * @param options - Options\n *\n * @returns Details observable\n */\nexport function watchDetails(\n  el: HTMLDetailsElement, { target$, print$ }: WatchOptions\n): Observable<Details> {\n  return target$\n    .pipe(\n      map(target => target.closest(\"details:not([open])\")!),\n      filter(details => el === details),\n      mergeWith(print$),\n      mapTo(el)\n    )\n}\n\n/**\n * Mount details\n *\n * This function ensures that `details` tags are opened on anchor jumps and\n * prior to printing, so the whole content of the page is visible.\n *\n * @param el - Details element\n * @param options - Options\n *\n * @returns Details component observable\n */\nexport function mountDetails(\n  el: HTMLDetailsElement, options: MountOptions\n): Observable<Component<Details>> {\n  const internal$ = new Subject<Details>()\n  internal$.subscribe(() => {\n    el.setAttribute(\"open\", \"\")\n    el.scrollIntoView()\n  })\n\n  /* Create and return component */\n  return watchDetails(el, options)\n    .pipe(\n      tap(internal$),\n      mapTo({ ref: el })\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Observable, of } from \"rxjs\"\n\nimport { createElement, replaceElement } from \"~/browser\"\nimport { renderTable } from \"~/templates\"\n\nimport { Component } from \"../../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Data table\n */\nexport interface DataTable {}\n\n/* ----------------------------------------------------------------------------\n * Data\n * ------------------------------------------------------------------------- */\n\n/**\n * Sentinel for replacement\n */\nconst sentinel = createElement(\"table\")\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount data table\n *\n * This function wraps a data table in another scrollable container, so it can\n * be smoothly scrolled on smaller screen sizes and won't break the layout.\n *\n * @param el - Data table element\n *\n * @returns Data table component observable\n */\nexport function mountDataTable(\n  el: HTMLElement\n): Observable<Component<DataTable>> {\n  replaceElement(el, sentinel)\n  replaceElement(sentinel, renderTable(el))\n\n  /* Create and return component */\n  return of({ ref: el })\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Observable, merge } from \"rxjs\"\n\nimport { Viewport, getElements } from \"~/browser\"\n\nimport { Component } from \"../../_\"\nimport { CodeBlock, mountCodeBlock } from \"../code\"\nimport { Details, mountDetails } from \"../details\"\nimport { DataTable, mountDataTable } from \"../table\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Content\n */\nexport type Content =\n  | CodeBlock\n  | DataTable\n  | Details\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  target$: Observable<HTMLElement>     /* Location target observable */\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  print$: Observable<void>             /* Print mode observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount content\n *\n * This function mounts all components that are found in the content of the\n * actual article, including code blocks, data tables and details.\n *\n * @param el - Content element\n * @param options - Options\n *\n * @returns Content component observable\n */\nexport function mountContent(\n  el: HTMLElement, { target$, viewport$, print$ }: MountOptions\n): Observable<Component<Content>> {\n  return merge(\n\n    /* Code blocks */\n    ...getElements(\"pre > code\", el)\n      .map(child => mountCodeBlock(child, { viewport$ })),\n\n    /* Data tables */\n    ...getElements(\"table:not([class])\", el)\n      .map(child => mountDataTable(child)),\n\n    /* Details */\n    ...getElements(\"details\", el)\n      .map(child => mountDetails(child, { target$, print$ }))\n  )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  animationFrameScheduler,\n  merge,\n  of\n} from \"rxjs\"\nimport {\n  delay,\n  map,\n  observeOn,\n  switchMap,\n  tap\n} from \"rxjs/operators\"\n\nimport {\n  resetDialogState,\n  setDialogMessage,\n  setDialogState\n} from \"~/actions\"\n\nimport { Component } from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Dialog\n */\nexport interface Dialog {\n  message: string                      /* Dialog message */\n  open: boolean                        /* Dialog is visible */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  alert$: Subject<string>              /* Alert subject */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  alert$: Subject<string>              /* Alert subject */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch dialog\n *\n * @param _el - Dialog element\n * @param options - Options\n *\n * @returns Dialog observable\n */\nexport function watchDialog(\n  _el: HTMLElement, { alert$ }: WatchOptions\n): Observable<Dialog> {\n  return alert$\n    .pipe(\n      switchMap(message => merge(\n        of(true),\n        of(false).pipe(delay(2000))\n      )\n        .pipe(\n          map(open => ({ message, open }))\n        )\n      )\n    )\n}\n\n/**\n * Mount dialog\n *\n * This function reveals the dialog in the right cornerwhen a new alert is\n * emitted through the subject that is passed as part of the options.\n *\n * @param el - Dialog element\n * @param options - Options\n *\n * @returns Dialog component observable\n */\nexport function mountDialog(\n  el: HTMLElement, options: MountOptions\n): Observable<Component<Dialog>> {\n  const internal$ = new Subject<Dialog>()\n  internal$\n    .pipe(\n      observeOn(animationFrameScheduler)\n    )\n      .subscribe(({ message, open }) => {\n        setDialogMessage(el, message)\n        if (open)\n          setDialogState(el, \"open\")\n        else\n          resetDialogState(el)\n      })\n\n  /* Create and return component */\n  return watchDialog(el, options)\n    .pipe(\n      tap(internal$),\n      map(state => ({ ref: el, ...state }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  animationFrameScheduler,\n  combineLatest,\n  defer,\n  of\n} from \"rxjs\"\nimport {\n  bufferCount,\n  combineLatestWith,\n  distinctUntilChanged,\n  distinctUntilKeyChanged,\n  filter,\n  map,\n  observeOn,\n  shareReplay,\n  startWith,\n  switchMap\n} from \"rxjs/operators\"\n\nimport { feature } from \"~/_\"\nimport { resetHeaderState, setHeaderState } from \"~/actions\"\nimport {\n  Viewport,\n  watchElementSize,\n  watchToggle\n} from \"~/browser\"\n\nimport { Component } from \"../../_\"\nimport { Main } from \"../../main\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Header\n */\nexport interface Header {\n  height: number                       /* Header visible height */\n  sticky: boolean                      /* Header stickyness */\n  hidden: boolean                      /* User scrolled past threshold */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n  main$: Observable<Main>              /* Main area observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Compute whether the header is hidden\n *\n * If the user scrolls past a certain threshold, the header can be hidden when\n * scrolling down, and shown when scrolling up.\n *\n * @param options - Options\n *\n * @returns Toggle observable\n */\nfunction isHidden({ viewport$ }: WatchOptions): Observable<boolean> {\n  if (!feature(\"header.autohide\"))\n    return of(false)\n\n  /* Compute direction and turning point */\n  const direction$ = viewport$\n    .pipe(\n      map(({ offset: { y } }) => y),\n      bufferCount(2, 1),\n      map(([a, b]) => [a < b, b] as const),\n      distinctUntilKeyChanged(0)\n    )\n\n  /* Compute whether header should be hidden */\n  const hidden$ = combineLatest([viewport$, direction$])\n    .pipe(\n      filter(([{ offset }, [, y]]) => Math.abs(y - offset.y) > 100),\n      map(([, [direction]]) => direction),\n      distinctUntilChanged()\n    )\n\n  /* Compute threshold for hiding */\n  const search$ = watchToggle(\"search\")\n  return combineLatest([viewport$, search$])\n    .pipe(\n      map(([{ offset }, search]) => offset.y > 400 && !search),\n      distinctUntilChanged(),\n      switchMap(active => active ? hidden$ : of(false)),\n      startWith(false)\n    )\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch header\n *\n * @param el - Header element\n * @param options - Options\n *\n * @returns Header observable\n */\nexport function watchHeader(\n  el: HTMLElement, options: WatchOptions\n): Observable<Header> {\n  return defer(() => {\n    const styles = getComputedStyle(el)\n    return of(\n      styles.position === \"sticky\" ||\n      styles.position === \"-webkit-sticky\"\n    )\n  })\n    .pipe(\n      combineLatestWith(watchElementSize(el), isHidden(options)),\n      map(([sticky, { height }, hidden]) => ({\n        height: sticky ? height : 0,\n        sticky,\n        hidden\n      })),\n      distinctUntilChanged((a, b) => (\n        a.sticky === b.sticky &&\n        a.height === b.height &&\n        a.hidden === b.hidden\n      )),\n      shareReplay(1)\n    )\n}\n\n/**\n * Mount header\n *\n * This function manages the different states of the header, i.e. whether it's\n * hidden or rendered with a shadow. This depends heavily on the main area.\n *\n * @param el - Header element\n * @param options - Options\n *\n * @returns Header component observable\n */\nexport function mountHeader(\n  el: HTMLElement, { header$, main$ }: MountOptions\n): Observable<Component<Header>> {\n  const internal$ = new Subject<Main>()\n  internal$\n    .pipe(\n      distinctUntilKeyChanged(\"active\"),\n      combineLatestWith(header$),\n      observeOn(animationFrameScheduler)\n    )\n      .subscribe(([{ active }, { hidden }]) => {\n        if (active)\n          setHeaderState(el, hidden ? \"hidden\" : \"shadow\")\n        else\n          resetHeaderState(el)\n      })\n\n  /* Connect to long-living subject and return component */\n  main$.subscribe(main => internal$.next(main))\n  return header$\n    .pipe(\n      map(state => ({ ref: el, ...state }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  NEVER,\n  Observable,\n  Subject,\n  animationFrameScheduler\n} from \"rxjs\"\nimport {\n  distinctUntilKeyChanged,\n  map,\n  observeOn,\n  tap\n} from \"rxjs/operators\"\n\nimport {\n  resetHeaderTitleState,\n  setHeaderTitleState\n} from \"~/actions\"\nimport {\n  Viewport,\n  getElement,\n  getElementSize,\n  watchViewportAt\n} from \"~/browser\"\n\nimport { Component } from \"../../_\"\nimport { Header } from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Header\n */\nexport interface HeaderTitle {\n  active: boolean                      /* User scrolled past first headline */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch header title\n *\n * @param el - Heading element\n * @param options - Options\n *\n * @returns Header title observable\n */\nexport function watchHeaderTitle(\n  el: HTMLHeadingElement, { viewport$, header$ }: WatchOptions\n): Observable<HeaderTitle> {\n  return watchViewportAt(el, { header$, viewport$ })\n    .pipe(\n      map(({ offset: { y } }) => {\n        const { height } = getElementSize(el)\n        return {\n          active: y >= height\n        }\n      }),\n      distinctUntilKeyChanged(\"active\")\n    )\n}\n\n/**\n * Mount header title\n *\n * This function swaps the header title from the site title to the title of the\n * current page when the user scrolls past the first headline.\n *\n * @param el - Header title element\n * @param options - Options\n *\n * @returns Header title component observable\n */\nexport function mountHeaderTitle(\n  el: HTMLElement, options: MountOptions\n): Observable<Component<HeaderTitle>> {\n  const internal$ = new Subject<HeaderTitle>()\n  internal$\n    .pipe(\n      observeOn(animationFrameScheduler)\n    )\n      .subscribe(({ active }) => {\n        if (active)\n          setHeaderTitleState(el, \"active\")\n        else\n          resetHeaderTitleState(el)\n      })\n\n  /* Obtain headline, if any */\n  const headline = getElement<HTMLHeadingElement>(\"article h1\")\n  if (typeof headline === \"undefined\")\n    return NEVER\n\n  /* Create and return component */\n  return watchHeaderTitle(headline, options)\n    .pipe(\n      tap(internal$),\n      map(state => ({ ref: el, ...state }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  combineLatest\n} from \"rxjs\"\nimport {\n  distinctUntilChanged,\n  distinctUntilKeyChanged,\n  map,\n  switchMap\n} from \"rxjs/operators\"\n\nimport { Viewport, watchElementSize } from \"~/browser\"\n\nimport { Header } from \"../header\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Main area\n */\nexport interface Main {\n  offset: number                       /* Main area top offset */\n  height: number                       /* Main area visible height */\n  active: boolean                      /* User scrolled past header */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch main area\n *\n * This function returns an observable that computes the visual parameters of\n * the main area which depends on the viewport vertical offset and height, as\n * well as the height of the header element, if the header is fixed.\n *\n * @param el - Main area element\n * @param options - Options\n *\n * @returns Main area observable\n */\nexport function watchMain(\n  el: HTMLElement, { viewport$, header$ }: WatchOptions\n): Observable<Main> {\n\n  /* Compute necessary adjustment for header */\n  const adjust$ = header$\n    .pipe(\n      map(({ height }) => height),\n      distinctUntilChanged()\n    )\n\n  /* Compute the main area's top and bottom borders */\n  const border$ = adjust$\n    .pipe(\n      switchMap(() => watchElementSize(el)\n        .pipe(\n          map(({ height }) => ({\n            top:    el.offsetTop,\n            bottom: el.offsetTop + height\n          })),\n          distinctUntilKeyChanged(\"bottom\")\n        )\n      )\n    )\n\n  /* Compute the main area's offset, visible height and if we scrolled past */\n  return combineLatest([adjust$, border$, viewport$])\n    .pipe(\n      map(([header, { top, bottom }, { offset: { y }, size: { height } }]) => {\n        height = Math.max(0, height\n          - Math.max(0, top    - y,  header)\n          - Math.max(0, height + y - bottom)\n        )\n        return {\n          offset: top - header,\n          height,\n          active: top - header <= y\n        }\n      }),\n      distinctUntilChanged((a, b) => (\n        a.offset === b.offset &&\n        a.height === b.height &&\n        a.active === b.active\n      ))\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  fromEvent,\n  of\n} from \"rxjs\"\nimport {\n  map,\n  mapTo,\n  mergeMap,\n  shareReplay,\n  startWith,\n  tap\n} from \"rxjs/operators\"\n\nimport { getElements } from \"~/browser\"\n\nimport { Component } from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Palette colors\n */\nexport interface PaletteColor {\n  scheme?: string                      /* Color scheme */\n  primary?: string                     /* Primary color */\n  accent?: string                      /* Accent color */\n}\n\n/**\n * Palette\n */\nexport interface Palette {\n  index: number                        /* Palette index */\n  color: PaletteColor                  /* Palette colors */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch color palette\n *\n * @param inputs - Color palette element\n *\n * @returns Color palette observable\n */\nexport function watchPalette(\n  inputs: HTMLInputElement[]\n): Observable<Palette> {\n  const data = localStorage.getItem(__prefix(\"__palette\"))!\n  const current = JSON.parse(data) || {\n    index: inputs.findIndex(input => (\n      matchMedia(input.getAttribute(\"data-md-color-media\")!).matches\n    ))\n  }\n\n  /* Emit changes in color palette */\n  const palette$ = of(...inputs)\n    .pipe(\n      mergeMap(input => fromEvent(input, \"change\")\n        .pipe(\n          mapTo(input)\n        )\n      ),\n      startWith(inputs[Math.max(0, current.index)]),\n      map(input => ({\n        index: inputs.indexOf(input),\n        color: {\n          scheme:  input.getAttribute(\"data-md-color-scheme\"),\n          primary: input.getAttribute(\"data-md-color-primary\"),\n          accent:  input.getAttribute(\"data-md-color-accent\")\n        }\n      } as Palette)),\n      shareReplay(1)\n    )\n\n  /* Persist preference in local storage */\n  palette$.subscribe(palette => {\n    localStorage.setItem(__prefix(\"__palette\"), JSON.stringify(palette))\n  })\n\n  /* Return palette */\n  return palette$\n}\n\n/**\n * Mount color palette\n *\n * @param el - Color palette element\n *\n * @returns Color palette component observable\n */\nexport function mountPalette(\n  el: HTMLElement\n): Observable<Component<Palette>> {\n  const internal$ = new Subject<Palette>()\n\n  /* Set color palette */\n  internal$.subscribe(palette => {\n    for (const [key, value] of Object.entries(palette.color))\n      if (typeof value === \"string\")\n        document.body.setAttribute(`data-md-color-${key}`, value)\n\n    /* Toggle visibility */\n    for (let index = 0; index < inputs.length; index++) {\n      const label = inputs[index].nextElementSibling\n      if (label instanceof HTMLElement)\n        label.hidden = palette.index !== index\n    }\n  })\n\n  /* Create and return component */\n  const inputs = getElements<HTMLInputElement>(\"input\", el)\n  return watchPalette(inputs)\n    .pipe(\n      tap(internal$),\n      map(state => ({ ref: el, ...state }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport ClipboardJS from \"clipboard\"\nimport { Observable, Subject } from \"rxjs\"\n\nimport { translation } from \"~/_\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Setup options\n */\ninterface SetupOptions {\n  alert$: Subject<string>              /* Alert subject */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set up Clipboard.js integration\n *\n * @param options - Options\n */\nexport function setupClipboardJS(\n  { alert$ }: SetupOptions\n): void {\n  if (ClipboardJS.isSupported()) {\n    new Observable<ClipboardJS.Event>(subscriber => {\n      new ClipboardJS(\"[data-clipboard-target], [data-clipboard-text]\")\n        .on(\"success\", ev => subscriber.next(ev))\n    })\n      .subscribe(() => alert$.next(translation(\"clipboard.copied\")))\n  }\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  EMPTY,\n  NEVER,\n  Observable,\n  Subject,\n  fromEvent,\n  merge,\n  of\n} from \"rxjs\"\nimport {\n  bufferCount,\n  catchError,\n  concatMap,\n  debounceTime,\n  distinctUntilChanged,\n  distinctUntilKeyChanged,\n  filter,\n  map,\n  sample,\n  share,\n  skip,\n  skipUntil,\n  switchMap\n} from \"rxjs/operators\"\n\nimport { configuration } from \"~/_\"\nimport {\n  Viewport,\n  ViewportOffset,\n  createElement,\n  getElement,\n  getElements,\n  replaceElement,\n  request,\n  requestXML,\n  setLocation,\n  setLocationHash,\n  setViewportOffset\n} from \"~/browser\"\nimport { getComponentElement } from \"~/components\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * History state\n */\nexport interface HistoryState {\n  url: URL                             /* State URL */\n  offset?: ViewportOffset              /* State viewport offset */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Setup options\n */\ninterface SetupOptions {\n  document$: Subject<Document>         /* Document subject */\n  location$: Subject<URL>              /* Location subject */\n  viewport$: Observable<Viewport>      /* Viewport observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Preprocess a list of URLs\n *\n * This function replaces the `site_url` in the sitemap with the actual base\n * URL, to allow instant loading to work in occasions like Netlify previews.\n *\n * @param urls - URLs\n *\n * @returns Processed URLs\n */\nfunction preprocess(urls: string[]): string[] {\n  if (urls.length < 2)\n    return urls\n\n  /* Take the first two URLs and remove everything after the last slash */\n  const [root, next] = urls\n    .sort((a, b) => a.length - b.length)\n    .map(url => url.replace(/[^/]+$/, \"\"))\n\n  /* Compute common prefix */\n  let index = 0\n  if (root === next)\n    index = root.length\n  else\n    while (root.charCodeAt(index) === next.charCodeAt(index))\n      index++\n\n  /* Replace common prefix (i.e. base) with effective base */\n  const config = configuration()\n  return urls.map(url => (\n    url.replace(root.slice(0, index), config.base)\n  ))\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set up instant loading\n *\n * When fetching, theoretically, we could use `responseType: \"document\"`, but\n * since all MkDocs links are relative, we need to make sure that the current\n * location matches the document we just loaded. Otherwise any relative links\n * in the document could use the old location.\n *\n * This is the reason why we need to synchronize history events and the process\n * of fetching the document for navigation changes (except `popstate` events):\n *\n * 1. Fetch document via `XMLHTTPRequest`\n * 2. Set new location via `history.pushState`\n * 3. Parse and emit fetched document\n *\n * For `popstate` events, we must not use `history.pushState`, or the forward\n * history will be irreversibly overwritten. In case the request fails, the\n * location change is dispatched regularly.\n *\n * @param options - Options\n */\nexport function setupInstantLoading(\n  { document$, location$, viewport$ }: SetupOptions\n): void {\n  const config = configuration()\n  if (location.protocol === \"file:\")\n    return\n\n  /* Disable automatic scroll restoration */\n  if (\"scrollRestoration\" in history) {\n    history.scrollRestoration = \"manual\"\n\n    /* Hack: ensure that reloads restore viewport offset */\n    fromEvent(window, \"beforeunload\")\n      .subscribe(() => {\n        history.scrollRestoration = \"auto\"\n      })\n  }\n\n  /* Hack: ensure absolute favicon link to omit 404s when switching */\n  const favicon = getElement<HTMLLinkElement>(\"link[rel=icon]\")\n  if (typeof favicon !== \"undefined\")\n    favicon.href = favicon.href\n\n  /* Intercept internal navigation */\n  const push$ = requestXML(new URL(\"sitemap.xml\", config.base))\n    .pipe(\n      map(sitemap => preprocess(getElements(\"loc\", sitemap)\n        .map(node => node.textContent!)\n      )),\n      switchMap(urls => fromEvent<MouseEvent>(document.body, \"click\")\n        .pipe(\n          filter(ev => !ev.metaKey && !ev.ctrlKey),\n          switchMap(ev => {\n\n            /* Handle HTML and SVG elements */\n            if (ev.target instanceof Element) {\n              const el = ev.target.closest(\"a\")\n              if (el && !el.target) {\n                const url = new URL(el.href)\n\n                /* Canonicalize URL */\n                url.search = \"\"\n                url.hash = \"\"\n\n                /* Check if URL should be intercepted */\n                if (\n                  url.pathname !== location.pathname &&\n                  urls.includes(url.toString())\n                ) {\n                  ev.preventDefault()\n                  return of({\n                    url: new URL(el.href)\n                  })\n                }\n              }\n            }\n            return NEVER\n          })\n        )\n      ),\n      share<HistoryState>()\n    )\n\n  /* Intercept history back and forward */\n  const pop$ = fromEvent<PopStateEvent>(window, \"popstate\")\n    .pipe(\n      filter(ev => ev.state !== null),\n      map(ev => ({\n        url: new URL(location.href),\n        offset: ev.state\n      })),\n      share<HistoryState>()\n    )\n\n  /* Emit location change */\n  merge(push$, pop$)\n    .pipe(\n      distinctUntilChanged((a, b) => a.url.href === b.url.href),\n      map(({ url }) => url)\n    )\n      .subscribe(location$)\n\n  /* Fetch document via `XMLHTTPRequest` */\n  const response$ = location$\n    .pipe(\n      distinctUntilKeyChanged(\"pathname\"),\n      switchMap(url => request(url.href)\n        .pipe(\n          catchError(() => {\n            setLocation(url)\n            return NEVER\n          })\n        )\n      ),\n      share()\n    )\n\n  /* Set new location via `history.pushState` */\n  push$\n    .pipe(\n      sample(response$)\n    )\n      .subscribe(({ url }) => {\n        history.pushState({}, \"\", `${url}`)\n      })\n\n  /* Parse and emit fetched document */\n  const dom = new DOMParser()\n  response$\n    .pipe(\n      switchMap(res => res.text()),\n      map(res => dom.parseFromString(res, \"text/html\"))\n    )\n      .subscribe(document$)\n\n  /* Replace meta tags and components */\n  document$\n    .pipe(\n      skip(1)\n    )\n      .subscribe(replacement => {\n        for (const selector of [\n\n          /* Meta tags */\n          \"title\",\n          \"link[rel=canonical]\",\n          \"meta[name=author]\",\n          \"meta[name=description]\",\n\n          /* Components */\n          \"[data-md-component=announce]\",\n          \"[data-md-component=container]\",\n          \"[data-md-component=header-topic]\",\n          \"[data-md-component=logo], .md-logo\", // compat\n          \"[data-md-component=skip]\"\n        ]) {\n          const source = getElement(selector)\n          const target = getElement(selector, replacement)\n          if (\n            typeof source !== \"undefined\" &&\n            typeof target !== \"undefined\"\n          ) {\n            replaceElement(source, target)\n          }\n        }\n      })\n\n  /* Re-evaluate scripts */\n  document$\n    .pipe(\n      skip(1),\n      map(() => getComponentElement(\"container\")),\n      switchMap(el => of(...getElements(\"script\", el))),\n      concatMap(el => {\n        const script = createElement(\"script\")\n        if (el.src) {\n          for (const name of el.getAttributeNames())\n            script.setAttribute(name, el.getAttribute(name)!)\n          replaceElement(el, script)\n\n          /* Complete when script is loaded */\n          return new Observable(observer => {\n            script.onload = () => observer.complete()\n          })\n\n        /* Complete immediately */\n        } else {\n          script.textContent = el.textContent\n          replaceElement(el, script)\n          return EMPTY\n        }\n      })\n    )\n      .subscribe()\n\n  /* Emit history state change */\n  merge(push$, pop$)\n    .pipe(\n      sample(document$),\n    )\n      .subscribe(({ url, offset }) => {\n        if (url.hash && !offset) {\n          setLocationHash(url.hash)\n        } else {\n          setViewportOffset(offset || { y: 0 })\n        }\n      })\n\n  /* Debounce update of viewport offset */\n  viewport$\n    .pipe(\n      skipUntil(push$),\n      debounceTime(250),\n      distinctUntilKeyChanged(\"offset\")\n    )\n      .subscribe(({ offset }) => {\n        history.replaceState(offset, \"\")\n      })\n\n  /* Set viewport offset from history */\n  merge(push$, pop$)\n    .pipe(\n      bufferCount(2, 1),\n      filter(([a, b]) => a.url.pathname === b.url.pathname),\n      map(([, state]) => state)\n    )\n      .subscribe(({ offset }) => {\n        setViewportOffset(offset || { y: 0 })\n      })\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport escapeHTML from \"escape-html\"\n\nimport { SearchIndexDocument } from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search document\n */\nexport interface SearchDocument extends SearchIndexDocument {\n  parent?: SearchIndexDocument         /* Parent article */\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Search document mapping\n */\nexport type SearchDocumentMap = Map<string, SearchDocument>\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Create a search document mapping\n *\n * @param docs - Search index documents\n *\n * @returns Search document map\n */\nexport function setupSearchDocumentMap(\n  docs: SearchIndexDocument[]\n): SearchDocumentMap {\n  const documents = new Map<string, SearchDocument>()\n  const parents   = new Set<SearchDocument>()\n  for (const doc of docs) {\n    const [path, hash] = doc.location.split(\"#\")\n\n    /* Extract location and title */\n    const location = doc.location\n    const title    = doc.title\n\n    /* Escape and cleanup text */\n    const text = escapeHTML(doc.text)\n      .replace(/\\s+(?=[,.:;!?])/g, \"\")\n      .replace(/\\s+/g, \" \")\n\n    /* Handle section */\n    if (hash) {\n      const parent = documents.get(path)!\n\n      /* Ignore first section, override article */\n      if (!parents.has(parent)) {\n        parent.title = doc.title\n        parent.text  = text\n\n        /* Remember that we processed the article */\n        parents.add(parent)\n\n      /* Add subsequent section */\n      } else {\n        documents.set(location, {\n          location,\n          title,\n          text,\n          parent\n        })\n      }\n\n    /* Add article */\n    } else {\n      documents.set(location, {\n        location,\n        title,\n        text\n      })\n    }\n  }\n  return documents\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport escapeHTML from \"escape-html\"\n\nimport { SearchIndexConfig } from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search highlight function\n *\n * @param value - Value\n *\n * @returns Highlighted value\n */\nexport type SearchHighlightFn = (value: string) => string\n\n/**\n * Search highlight factory function\n *\n * @param query - Query value\n *\n * @returns Search highlight function\n */\nexport type SearchHighlightFactoryFn = (query: string) => SearchHighlightFn\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Create a search highlighter\n *\n * @param config - Search index configuration\n *\n * @returns Search highlight factory function\n */\nexport function setupSearchHighlighter(\n  config: SearchIndexConfig\n): SearchHighlightFactoryFn {\n  const separator = new RegExp(config.separator, \"img\")\n  const highlight = (_: unknown, data: string, term: string) => {\n    return `${data}<mark data-md-highlight>${term}</mark>`\n  }\n\n  /* Return factory function */\n  return (query: string) => {\n    query = query\n      .replace(/[\\s*+\\-:~^]+/g, \" \")\n      .trim()\n\n    /* Create search term match expression */\n    const match = new RegExp(`(^|${config.separator})(${\n      query\n        .replace(/[|\\\\{}()[\\]^$+*?.-]/g, \"\\\\$&\")\n        .replace(separator, \"|\")\n    })`, \"img\")\n\n    /* Highlight string value */\n    return value => escapeHTML(value)\n      .replace(match, highlight)\n      .replace(/<\\/mark>(\\s+)<mark[^>]*>/img, \"$1\")\n  }\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search transformation function\n *\n * @param value - Query value\n *\n * @returns Transformed query value\n */\nexport type SearchTransformFn = (value: string) => string\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Default transformation function\n *\n * 1. Search for terms in quotation marks and prepend a `+` modifier to denote\n *    that the resulting document must contain all terms, converting the query\n *    to an `AND` query (as opposed to the default `OR` behavior). While users\n *    may expect terms enclosed in quotation marks to map to span queries, i.e.\n *    for which order is important, Lunr.js doesn't support them, so the best\n *    we can do is to convert the terms to an `AND` query.\n *\n * 2. Replace control characters which are not located at the beginning of the\n *    query or preceded by white space, or are not followed by a non-whitespace\n *    character or are at the end of the query string. Furthermore, filter\n *    unmatched quotation marks.\n *\n * 3. Trim excess whitespace from left and right.\n *\n * @param query - Query value\n *\n * @returns Transformed query value\n */\nexport function defaultTransform(query: string): string {\n  return query\n    .split(/\"([^\"]+)\"/g)                            /* => 1 */\n      .map((terms, index) => index & 1\n        ? terms.replace(/^\\b|^(?![^\\x00-\\x7F]|$)|\\s+/g, \" +\")\n        : terms\n      )\n      .join(\"\")\n    .replace(/\"|(?:^|\\s+)[*+\\-:^~]+(?=\\s+|$)/g, \"\") /* => 2 */\n    .trim()                                         /* => 3 */\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A RTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { SearchIndex, SearchResult } from \"../../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search message type\n */\nexport const enum SearchMessageType {\n  SETUP,                               /* Search index setup */\n  READY,                               /* Search index ready */\n  QUERY,                               /* Search query */\n  RESULT                               /* Search results */\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * A message containing the data necessary to setup the search index\n */\nexport interface SearchSetupMessage {\n  type: SearchMessageType.SETUP        /* Message type */\n  data: SearchIndex                    /* Message data */\n}\n\n/**\n * A message indicating the search index is ready\n */\nexport interface SearchReadyMessage {\n  type: SearchMessageType.READY        /* Message type */\n}\n\n/**\n * A message containing a search query\n */\nexport interface SearchQueryMessage {\n  type: SearchMessageType.QUERY        /* Message type */\n  data: string                         /* Message data */\n}\n\n/**\n * A message containing results for a search query\n */\nexport interface SearchResultMessage {\n  type: SearchMessageType.RESULT       /* Message type */\n  data: SearchResult                   /* Message data */\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * A message exchanged with the search worker\n */\nexport type SearchMessage =\n  | SearchSetupMessage\n  | SearchReadyMessage\n  | SearchQueryMessage\n  | SearchResultMessage\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Type guard for search setup messages\n *\n * @param message - Search worker message\n *\n * @returns Test result\n */\nexport function isSearchSetupMessage(\n  message: SearchMessage\n): message is SearchSetupMessage {\n  return message.type === SearchMessageType.SETUP\n}\n\n/**\n * Type guard for search ready messages\n *\n * @param message - Search worker message\n *\n * @returns Test result\n */\nexport function isSearchReadyMessage(\n  message: SearchMessage\n): message is SearchReadyMessage {\n  return message.type === SearchMessageType.READY\n}\n\n/**\n * Type guard for search query messages\n *\n * @param message - Search worker message\n *\n * @returns Test result\n */\nexport function isSearchQueryMessage(\n  message: SearchMessage\n): message is SearchQueryMessage {\n  return message.type === SearchMessageType.QUERY\n}\n\n/**\n * Type guard for search result messages\n *\n * @param message - Search worker message\n *\n * @returns Test result\n */\nexport function isSearchResultMessage(\n  message: SearchMessage\n): message is SearchResultMessage {\n  return message.type === SearchMessageType.RESULT\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A RTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { ObservableInput, Subject, from } from \"rxjs\"\nimport { map, share } from \"rxjs/operators\"\n\nimport { configuration, feature, translation } from \"~/_\"\nimport { WorkerHandler, watchWorker } from \"~/browser\"\n\nimport { SearchIndex } from \"../../_\"\nimport {\n  SearchOptions,\n  SearchPipeline\n} from \"../../options\"\nimport {\n  SearchMessage,\n  SearchMessageType,\n  SearchSetupMessage,\n  isSearchResultMessage\n} from \"../message\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search worker\n */\nexport type SearchWorker = WorkerHandler<SearchMessage>\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set up search index\n *\n * @param data - Search index\n *\n * @returns Search index\n */\nfunction setupSearchIndex(\n  { config, docs, index }: SearchIndex\n): SearchIndex {\n\n  /* Override default language with value from translation */\n  if (config.lang.length === 1 && config.lang[0] === \"en\")\n    config.lang = [\n      translation(\"search.config.lang\")\n    ]\n\n  /* Override default separator with value from translation */\n  if (config.separator === \"[\\\\s\\\\-]+\")\n    config.separator = translation(\"search.config.separator\")\n\n  /* Set pipeline from translation */\n  const pipeline = translation(\"search.config.pipeline\")\n    .split(/\\s*,\\s*/)\n    .filter(Boolean) as SearchPipeline\n\n  /* Determine search options */\n  const options: SearchOptions = {\n    pipeline,\n    suggestions: feature(\"search.suggest\")\n  }\n\n  /* Return search index after defaulting */\n  return { config, docs, index, options }\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set up search worker\n *\n * This function creates a web worker to set up and query the search index,\n * which is done using Lunr.js. The index must be passed as an observable to\n * enable hacks like _localsearch_ via search index embedding as JSON.\n *\n * @param url - Worker URL\n * @param index - Search index observable input\n *\n * @returns Search worker\n */\nexport function setupSearchWorker(\n  url: string, index: ObservableInput<SearchIndex>\n): SearchWorker {\n  const config = configuration()\n  const worker = new Worker(url)\n\n  /* Create communication channels and resolve relative links */\n  const tx$ = new Subject<SearchMessage>()\n  const rx$ = watchWorker(worker, { tx$ })\n    .pipe(\n      map(message => {\n        if (isSearchResultMessage(message)) {\n          for (const result of message.data.items)\n            for (const document of result)\n              document.location = `${new URL(document.location, config.base)}`\n        }\n        return message\n      }),\n      share()\n    )\n\n  /* Set up search index */\n  from(index)\n    .pipe(\n      map<SearchIndex, SearchSetupMessage>(data => ({\n        type: SearchMessageType.SETUP,\n        data: setupSearchIndex(data)\n      }))\n    )\n      .subscribe(tx$.next.bind(tx$))\n\n  /* Return search worker */\n  return { tx$, rx$ }\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { configuration } from \"~/_\"\nimport { getElementOrThrow, requestJSON } from \"~/browser\"\nimport { Version, renderVersionSelector } from \"~/templates\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Set up version selector\n */\nexport function setupVersionSelector(): void {\n  const config = configuration()\n  requestJSON<Version[]>(new URL(\"../versions.json\", config.base))\n    .subscribe(versions => {\n      const topic = getElementOrThrow(\".md-header__topic\")\n      topic.appendChild(renderVersionSelector(versions))\n    })\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  combineLatest,\n  fromEvent,\n  merge\n} from \"rxjs\"\nimport {\n  delay,\n  distinctUntilChanged,\n  distinctUntilKeyChanged,\n  filter,\n  map,\n  take,\n  takeLast,\n  takeUntil,\n  tap\n} from \"rxjs/operators\"\n\nimport {\n  resetSearchQueryPlaceholder,\n  setSearchQueryPlaceholder\n} from \"~/actions\"\nimport {\n  getLocation,\n  setElementFocus,\n  setToggle,\n  watchElementFocus\n} from \"~/browser\"\nimport {\n  SearchMessageType,\n  SearchQueryMessage,\n  SearchWorker,\n  defaultTransform,\n  isSearchReadyMessage\n} from \"~/integrations\"\n\nimport { Component } from \"../../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search query\n */\nexport interface SearchQuery {\n  value: string                        /* Query value */\n  focus: boolean                       /* Query focus */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch search query\n *\n * Note that the focus event which triggers re-reading the current query value\n * is delayed by `1ms` so the input's empty state is allowed to propagate.\n *\n * @param el - Search query element\n * @param worker - Search worker\n *\n * @returns Search query observable\n */\nexport function watchSearchQuery(\n  el: HTMLInputElement, { rx$ }: SearchWorker\n): Observable<SearchQuery> {\n  const fn = __search?.transform || defaultTransform\n\n  /* Intercept focus and input events */\n  const focus$ = watchElementFocus(el)\n  const value$ = merge(\n    fromEvent(el, \"keyup\"),\n    fromEvent(el, \"focus\").pipe(delay(1))\n  )\n    .pipe(\n      map(() => fn(el.value)),\n      distinctUntilChanged()\n    )\n\n  /* Intercept deep links */\n  const location = getLocation()\n  if (location.searchParams.has(\"q\")) {\n    setToggle(\"search\", true)\n    rx$\n      .pipe(\n        filter(isSearchReadyMessage),\n        take(1)\n      )\n        .subscribe(() => {\n          el.value = location.searchParams.get(\"q\")!\n          setElementFocus(el)\n        })\n  }\n\n  /* Combine into single observable */\n  return combineLatest([value$, focus$])\n    .pipe(\n      map(([value, focus]) => ({ value, focus }))\n    )\n}\n\n/**\n * Mount search query\n *\n * @param el - Search query element\n * @param worker - Search worker\n *\n * @returns Search query component observable\n */\nexport function mountSearchQuery(\n  el: HTMLInputElement, { tx$, rx$ }: SearchWorker\n): Observable<Component<SearchQuery, HTMLInputElement>> {\n  const internal$ = new Subject<SearchQuery>()\n\n  /* Handle value changes */\n  internal$\n    .pipe(\n      distinctUntilKeyChanged(\"value\"),\n      map(({ value }): SearchQueryMessage => ({\n        type: SearchMessageType.QUERY,\n        data: value\n      }))\n    )\n      .subscribe(tx$.next.bind(tx$))\n\n  /* Handle focus changes */\n  internal$\n    .pipe(\n      distinctUntilKeyChanged(\"focus\")\n    )\n      .subscribe(({ focus }) => {\n        if (focus) {\n          setToggle(\"search\", focus)\n          setSearchQueryPlaceholder(el, \"\")\n        } else {\n          resetSearchQueryPlaceholder(el)\n        }\n      })\n\n  /* Handle reset */\n  fromEvent(el.form!, \"reset\")\n    .pipe(\n      takeUntil(internal$.pipe(takeLast(1)))\n    )\n      .subscribe(() => setElementFocus(el))\n\n  /* Create and return component */\n  return watchSearchQuery(el, { tx$, rx$ })\n    .pipe(\n      tap(internal$),\n      map(state => ({ ref: el, ...state }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  animationFrameScheduler,\n  merge,\n  of\n} from \"rxjs\"\nimport {\n  bufferCount,\n  filter,\n  map,\n  observeOn,\n  switchMap,\n  take,\n  tap,\n  withLatestFrom,\n  zipWith\n} from \"rxjs/operators\"\n\nimport {\n  addToSearchResultList,\n  resetSearchResultList,\n  resetSearchResultMeta,\n  setSearchResultMeta\n} from \"~/actions\"\nimport {\n  getElementOrThrow,\n  watchElementThreshold\n} from \"~/browser\"\nimport {\n  SearchResult,\n  SearchWorker,\n  isSearchReadyMessage,\n  isSearchResultMessage\n} from \"~/integrations\"\nimport { renderSearchResultItem } from \"~/templates\"\n\nimport { Component } from \"../../_\"\nimport { SearchQuery } from \"../query\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  query$: Observable<SearchQuery>      /* Search query observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount search result list\n *\n * This function performs a lazy rendering of the search results, depending on\n * the vertical offset of the search result container.\n *\n * @param el - Search result list element\n * @param worker - Search worker\n * @param options - Options\n *\n * @returns Search result list component observable\n */\nexport function mountSearchResult(\n  el: HTMLElement, { rx$ }: SearchWorker, { query$ }: MountOptions\n): Observable<Component<SearchResult>> {\n  const internal$ = new Subject<SearchResult>()\n  const boundary$ = watchElementThreshold(el.parentElement!)\n    .pipe(\n      filter(Boolean)\n    )\n\n  /* Retrieve nested components */\n  const meta = getElementOrThrow(\":scope > :first-child\", el)\n  const list = getElementOrThrow(\":scope > :last-child\", el)\n\n  /* Update search result metadata when ready */\n  rx$\n    .pipe(\n      filter(isSearchReadyMessage),\n      take(1)\n    )\n      .subscribe(() => {\n        resetSearchResultMeta(meta)\n      })\n\n  /* Update search result metadata */\n  internal$\n    .pipe(\n      observeOn(animationFrameScheduler),\n      withLatestFrom(query$)\n    )\n      .subscribe(([{ items }, { value }]) => {\n        if (value)\n          setSearchResultMeta(meta, items.length)\n        else\n          resetSearchResultMeta(meta)\n      })\n\n  /* Update search result list */\n  internal$\n    .pipe(\n      observeOn(animationFrameScheduler),\n      tap(() => resetSearchResultList(list)),\n      switchMap(({ items }) => merge(\n        of(...items.slice(0, 10)),\n        of(...items.slice(10))\n          .pipe(\n            bufferCount(4),\n            zipWith(boundary$),\n            switchMap(([chunk]) => of(...chunk))\n          )\n      ))\n    )\n      .subscribe(result => {\n        addToSearchResultList(list, renderSearchResultItem(result))\n      })\n\n  /* Filter search result message */\n  const result$ = rx$\n    .pipe(\n      filter(isSearchResultMessage),\n      map(({ data }) => data)\n    )\n\n  /* Create and return component */\n  return result$\n    .pipe(\n      tap(internal$),\n      map(state => ({ ref: el, ...state }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  fromEvent\n} from \"rxjs\"\nimport { map, tap } from \"rxjs/operators\"\n\nimport { getLocation } from \"~/browser\"\n\nimport { Component } from \"../../_\"\nimport { SearchQuery } from \"../query\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search sharing\n */\nexport interface SearchShare {\n  url: URL                             /* Deep link for sharing */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  query$: Observable<SearchQuery>      /* Search query observable */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  query$: Observable<SearchQuery>      /* Search query observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount search sharing\n *\n * @param _el - Search sharing element\n * @param options - Options\n *\n * @returns Search sharing observable\n */\nexport function watchSearchShare(\n  _el: HTMLElement, { query$ }: WatchOptions\n): Observable<SearchShare> {\n  return query$\n    .pipe(\n      map(({ value }) => {\n        const url = getLocation()\n        url.hash = \"\"\n        url.searchParams.delete(\"h\")\n        url.searchParams.set(\"q\", value)\n        return { url }\n      })\n    )\n}\n\n/**\n * Mount search sharing\n *\n * @param el - Search sharing element\n * @param options - Options\n *\n * @returns Search sharing component observable\n */\nexport function mountSearchShare(\n  el: HTMLAnchorElement, options: MountOptions\n): Observable<Component<SearchShare>> {\n  const internal$ = new Subject<SearchShare>()\n  internal$.subscribe(({ url }) => {\n    el.setAttribute(\"data-clipboard-text\", el.href)\n    el.href = `${url}`\n  })\n\n  /* Prevent following of link */\n  fromEvent(el, \"click\")\n    .subscribe(ev => ev.preventDefault())\n\n  /* Create and return component */\n  return watchSearchShare(el, options)\n    .pipe(\n      tap(internal$),\n      map(state => ({ ref: el, ...state }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  asyncScheduler,\n  fromEvent\n} from \"rxjs\"\nimport {\n  combineLatestWith,\n  distinctUntilChanged,\n  filter,\n  map,\n  observeOn,\n  tap\n} from \"rxjs/operators\"\n\nimport { Keyboard } from \"~/browser\"\nimport {\n  SearchResult,\n  SearchWorker,\n  isSearchResultMessage\n} from \"~/integrations\"\n\nimport { Component, getComponentElement } from \"../../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search suggestions\n */\nexport interface SearchSuggest {}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  keyboard$: Observable<Keyboard>      /* Keyboard observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount search suggestions\n *\n * This function will perform a lazy rendering of the search results, depending\n * on the vertical offset of the search result container.\n *\n * @param el - Search result list element\n * @param worker - Search worker\n * @param options - Options\n *\n * @returns Search result list component observable\n */\nexport function mountSearchSuggest(\n  el: HTMLElement, { rx$ }: SearchWorker, { keyboard$ }: MountOptions\n): Observable<Component<SearchSuggest>> {\n  const internal$ = new Subject<SearchResult>()\n\n  /* Retrieve query component and track all changes */\n  const query  = getComponentElement(\"search-query\")\n  const query$ = fromEvent(query, \"keydown\")\n    .pipe(\n      observeOn(asyncScheduler),\n      map(() => query.value),\n      distinctUntilChanged(),\n    )\n\n  /* Update search suggestions */\n  internal$\n    .pipe(\n      combineLatestWith(query$),\n      map(([{ suggestions }, value]) => {\n        const words = value.split(/([\\s-]+)/)\n        if (suggestions?.length && words[words.length - 1]) {\n          const last = suggestions[suggestions.length - 1]\n          if (last.startsWith(words[words.length - 1]))\n            words[words.length - 1] = last\n        } else {\n          words.length = 0\n        }\n        return words\n      })\n    )\n      .subscribe(words => el.innerHTML = words\n        .join(\"\")\n        .replace(/\\s/g, \"&nbsp;\")\n      )\n\n  /* Set up search keyboard handlers */\n  keyboard$\n    .pipe(\n      filter(({ mode }) => mode === \"search\")\n    )\n      .subscribe(key => {\n        switch (key.type) {\n\n          /* Right arrow: accept current suggestion */\n          case \"ArrowRight\":\n            if (\n              el.innerText.length &&\n              query.selectionStart === query.value.length\n            )\n              query.value = el.innerText\n            break\n        }\n      })\n\n  /* Filter search result message */\n  const result$ = rx$\n    .pipe(\n      filter(isSearchResultMessage),\n      map(({ data }) => data)\n    )\n\n  /* Create and return component */\n  return result$\n    .pipe(\n      tap(internal$),\n      map(() => ({ ref: el }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { NEVER, Observable, ObservableInput, merge } from \"rxjs\"\nimport { filter, mergeWith, sample, take } from \"rxjs/operators\"\n\nimport { configuration } from \"~/_\"\nimport {\n  Keyboard,\n  getActiveElement,\n  getElements,\n  setElementFocus,\n  setElementSelection,\n  setToggle\n} from \"~/browser\"\nimport {\n  SearchIndex,\n  SearchResult,\n  isSearchQueryMessage,\n  isSearchReadyMessage,\n  setupSearchWorker\n} from \"~/integrations\"\n\nimport {\n  Component,\n  getComponentElement,\n  getComponentElements\n} from \"../../_\"\nimport { SearchQuery, mountSearchQuery } from \"../query\"\nimport { mountSearchResult } from \"../result\"\nimport { SearchShare, mountSearchShare } from \"../share\"\nimport { SearchSuggest, mountSearchSuggest } from \"../suggest\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search\n */\nexport type Search =\n  | SearchQuery\n  | SearchResult\n  | SearchShare\n  | SearchSuggest\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  index$: ObservableInput<SearchIndex> /* Search index observable */\n  keyboard$: Observable<Keyboard>      /* Keyboard observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount search\n *\n * This function sets up the search functionality, including the underlying\n * web worker and all keyboard bindings.\n *\n * @param el - Search element\n * @param options - Options\n *\n * @returns Search component observable\n */\nexport function mountSearch(\n  el: HTMLElement, { index$, keyboard$ }: MountOptions\n): Observable<Component<Search>> {\n  const config = configuration()\n  try {\n    const worker = setupSearchWorker(config.search, index$)\n\n    /* Retrieve query and result components */\n    const query  = getComponentElement(\"search-query\", el)\n    const result = getComponentElement(\"search-result\", el)\n\n    /* Re-emit query when search is ready */\n    const { tx$, rx$ } = worker\n    tx$\n      .pipe(\n        filter(isSearchQueryMessage),\n        sample(rx$\n          .pipe(\n            filter(isSearchReadyMessage),\n            take(1)\n          )\n        )\n      )\n        .subscribe(tx$.next.bind(tx$))\n\n    /* Set up search keyboard handlers */\n    keyboard$\n      .pipe(\n        filter(({ mode }) => mode === \"search\")\n      )\n        .subscribe(key => {\n          const active = getActiveElement()\n          switch (key.type) {\n\n            /* Enter: go to first (best) result */\n            case \"Enter\":\n              if (active === query) {\n                const anchors = new Map<HTMLAnchorElement, number>()\n                for (const anchor of getElements<HTMLAnchorElement>(\n                  \":first-child [href]\", result\n                )) {\n                  const article = anchor.firstElementChild!\n                  anchors.set(anchor, parseFloat(\n                    article.getAttribute(\"data-md-score\")!\n                  ))\n                }\n\n                /* Go to result with highest score, if any */\n                if (anchors.size) {\n                  const [[best]] = [...anchors].sort(([, a], [, b]) => b - a)\n                  best.click()\n                }\n\n                /* Otherwise omit form submission */\n                key.claim()\n              }\n              break\n\n            /* Escape or Tab: close search */\n            case \"Escape\":\n            case \"Tab\":\n              setToggle(\"search\", false)\n              setElementFocus(query, false)\n              break\n\n            /* Vertical arrows: select previous or next search result */\n            case \"ArrowUp\":\n            case \"ArrowDown\":\n              if (typeof active === \"undefined\") {\n                setElementFocus(query)\n              } else {\n                const els = [query, ...getElements(\n                  \":not(details) > [href], summary, details[open] [href]\",\n                  result\n                )]\n                const i = Math.max(0, (\n                  Math.max(0, els.indexOf(active)) + els.length + (\n                    key.type === \"ArrowUp\" ? -1 : +1\n                  )\n                ) % els.length)\n                setElementFocus(els[i])\n              }\n\n              /* Prevent scrolling of page */\n              key.claim()\n              break\n\n            /* All other keys: hand to search query */\n            default:\n              if (query !== getActiveElement())\n                setElementFocus(query)\n          }\n        })\n\n    /* Set up global keyboard handlers */\n    keyboard$\n      .pipe(\n        filter(({ mode }) => mode === \"global\"),\n      )\n        .subscribe(key => {\n          switch (key.type) {\n\n            /* Open search and select query */\n            case \"f\":\n            case \"s\":\n            case \"/\":\n              setElementFocus(query)\n              setElementSelection(query)\n              key.claim()\n              break\n          }\n        })\n\n    /* Create and return component */\n    const query$  = mountSearchQuery(query, worker)\n    const result$ = mountSearchResult(result, worker, { query$ })\n    return merge(query$, result$)\n      .pipe(\n        mergeWith(\n\n          /* Search sharing */\n          ...getComponentElements(\"search-share\", el)\n          .map(child => mountSearchShare(child, { query$ })),\n\n          /* Search suggestions */\n          ...getComponentElements(\"search-suggest\", el)\n            .map(child => mountSearchSuggest(child, worker, { keyboard$ }))\n        )\n      )\n\n  /* Gracefully handle broken search */\n  } catch (err) {\n    el.hidden = true\n    return NEVER\n  }\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  ObservableInput,\n  combineLatest\n} from \"rxjs\"\nimport { filter, map, startWith } from \"rxjs/operators\"\n\nimport { getLocation } from \"~/browser\"\nimport {\n  SearchIndex,\n  setupSearchHighlighter\n} from \"~/integrations\"\nimport { h } from \"~/utilities\"\n\nimport { Component } from \"../../_\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search highlighting\n */\nexport interface SearchHighlight {\n  nodes: Map<ChildNode, string>        /* Map of replacements */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  index$: ObservableInput<SearchIndex> /* Search index observable */\n  location$: Observable<URL>           /* Location observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Mount search highlighting\n *\n * @param el - Content element\n * @param options - Options\n *\n * @returns Search highlighting component observable\n */\nexport function mountSearchHiglight(\n  el: HTMLElement, { index$, location$ }: MountOptions\n): Observable<Component<SearchHighlight>> {\n  return combineLatest([\n    index$,\n    location$\n      .pipe(\n        startWith(getLocation()),\n        filter(url => url.searchParams.has(\"h\"))\n      )\n  ])\n    .pipe(\n      map(([index, url]) => setupSearchHighlighter(index.config)(\n        url.searchParams.get(\"h\")!\n      )),\n      map(fn => {\n        const nodes = new Map<ChildNode, string>()\n\n        /* Traverse text nodes and collect matches */\n        const it = document.createNodeIterator(el, NodeFilter.SHOW_TEXT)\n        for (let node = it.nextNode(); node; node = it.nextNode()) {\n          if (node.parentElement?.offsetHeight) {\n            const original = node.textContent!\n            const replaced = fn(original)\n            if (replaced.length > original.length)\n              nodes.set(node as ChildNode, replaced)\n          }\n        }\n\n        /* Replace original nodes with matches */\n        for (const [node, text] of nodes) {\n          const { childNodes } = h(\"span\", null, text)\n          node.replaceWith(...Array.from(childNodes))\n        }\n\n        /* Return component */\n        return { ref: el, nodes }\n      })\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  animationFrameScheduler,\n  combineLatest\n} from \"rxjs\"\nimport {\n  distinctUntilChanged,\n  map,\n  observeOn,\n  tap,\n  withLatestFrom\n} from \"rxjs/operators\"\n\nimport {\n  resetSidebarHeight,\n  resetSidebarOffset,\n  setSidebarHeight,\n  setSidebarOffset\n} from \"~/actions\"\nimport { Viewport } from \"~/browser\"\n\nimport { Component } from \"../_\"\nimport { Header } from \"../header\"\nimport { Main } from \"../main\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Sidebar\n */\nexport interface Sidebar {\n  height: number                       /* Sidebar height */\n  locked: boolean                      /* User scrolled past header */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  main$: Observable<Main>              /* Main area observable */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n  main$: Observable<Main>              /* Main area observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch sidebar\n *\n * This function returns an observable that computes the visual parameters of\n * the sidebar which depends on the vertical viewport offset, as well as the\n * height of the main area. When the page is scrolled beyond the header, the\n * sidebar is locked and fills the remaining space.\n *\n * @param el - Sidebar element\n * @param options - Options\n *\n * @returns Sidebar observable\n */\nexport function watchSidebar(\n  el: HTMLElement, { viewport$, main$ }: WatchOptions\n): Observable<Sidebar> {\n  const adjust =\n    el.parentElement!.offsetTop -\n    el.parentElement!.parentElement!.offsetTop\n\n  /* Compute the sidebar's available height and if it should be locked */\n  return combineLatest([main$, viewport$])\n    .pipe(\n      map(([{ offset, height }, { offset: { y } }]) => {\n        height = height\n          + Math.min(adjust, Math.max(0, y - offset))\n          - adjust\n        return {\n          height,\n          locked: y >= offset + adjust\n        }\n      }),\n      distinctUntilChanged((a, b) => (\n        a.height === b.height &&\n        a.locked === b.locked\n      ))\n    )\n}\n\n/**\n * Mount sidebar\n *\n * @param el - Sidebar element\n * @param options - Options\n *\n * @returns Sidebar component observable\n */\nexport function mountSidebar(\n  el: HTMLElement, { header$, ...options }: MountOptions\n): Observable<Component<Sidebar>> {\n  const internal$ = new Subject<Sidebar>()\n  internal$\n    .pipe(\n      observeOn(animationFrameScheduler),\n      withLatestFrom(header$)\n    )\n      .subscribe({\n\n        /* Update height and offset */\n        next([{ height }, { height: offset }]) {\n          setSidebarHeight(el, height)\n          setSidebarOffset(el, offset)\n        },\n\n        /* Reset on complete */\n        complete() {\n          resetSidebarOffset(el)\n          resetSidebarHeight(el)\n        }\n      })\n\n  /* Create and return component */\n  return watchSidebar(el, options)\n    .pipe(\n      tap(internal$),\n      map(state => ({ ref: el, ...state }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Repo, User } from \"github-types\"\nimport { Observable, zip } from \"rxjs\"\nimport { defaultIfEmpty, map } from \"rxjs/operators\"\n\nimport { requestJSON } from \"~/browser\"\n\nimport { SourceFacts } from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * GitHub release (partial)\n */\ninterface Release {\n  tag_name: string                     /* Tag name */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Fetch GitHub repository facts\n *\n * @param user - GitHub user\n * @param repo - GitHub repository\n *\n * @returns Repository facts observable\n */\nexport function fetchSourceFactsFromGitHub(\n  user: string, repo?: string\n): Observable<SourceFacts> {\n  if (typeof repo !== \"undefined\") {\n    const url = `https://api.github.com/repos/${user}/${repo}`\n    return zip(\n\n      /* Fetch version */\n      requestJSON<Release>(`${url}/releases/latest`)\n        .pipe(\n          map(release => ({\n            version: release.tag_name\n          })),\n          defaultIfEmpty({})\n        ),\n\n      /* Fetch stars and forks */\n      requestJSON<Repo>(url)\n        .pipe(\n          map(info => ({\n            stars: info.stargazers_count,\n            forks: info.forks_count\n          })),\n          defaultIfEmpty({})\n        )\n    )\n      .pipe(\n        map(([release, info]) => ({ ...release, ...info }))\n      )\n\n  /* User or organization */\n  } else {\n    const url = `https://api.github.com/repos/${user}`\n    return requestJSON<User>(url)\n      .pipe(\n        map(info => ({\n          repositories: info.public_repos\n        })),\n        defaultIfEmpty({})\n      )\n  }\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { ProjectSchema } from \"gitlab\"\nimport { Observable } from \"rxjs\"\nimport { defaultIfEmpty, map } from \"rxjs/operators\"\n\nimport { requestJSON } from \"~/browser\"\n\nimport { SourceFacts } from \"../_\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Fetch GitLab repository facts\n *\n * @param base - GitLab base\n * @param project - GitLab project\n *\n * @returns Repository facts observable\n */\nexport function fetchSourceFactsFromGitLab(\n  base: string, project: string\n): Observable<SourceFacts> {\n  const url = `https://${base}/api/v4/projects/${encodeURIComponent(project)}`\n  return requestJSON<ProjectSchema>(url)\n    .pipe(\n      map(({ star_count, forks_count }) => ({\n        stars: star_count,\n        forks: forks_count\n      })),\n      defaultIfEmpty({})\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { NEVER, Observable } from \"rxjs\"\n\nimport { fetchSourceFactsFromGitHub } from \"../github\"\nimport { fetchSourceFactsFromGitLab } from \"../gitlab\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Repository facts for repositories\n */\nexport interface RepositoryFacts {\n  stars?: number                       /* Number of stars */\n  forks?: number                       /* Number of forks */\n  version?: string                     /* Latest version */\n}\n\n/**\n * Repository facts for organizations\n */\nexport interface OrganizationFacts {\n  repositories?: number                /* Number of repositories */\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Repository facts\n */\nexport type SourceFacts =\n  | RepositoryFacts\n  | OrganizationFacts\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Fetch repository facts\n *\n * @param url - Repository URL\n *\n * @returns Repository facts observable\n */\nexport function fetchSourceFacts(\n  url: string\n): Observable<SourceFacts> {\n  const [type] = url.match(/(git(?:hub|lab))/i) || []\n  switch (type.toLowerCase()) {\n\n    /* GitHub repository */\n    case \"github\":\n      const [, user, repo] = url.match(/^.+github\\.com\\/([^/]+)\\/?([^/]+)?/i)!\n      return fetchSourceFactsFromGitHub(user, repo)\n\n    /* GitLab repository */\n    case \"gitlab\":\n      const [, base, slug] = url.match(/^.+?([^/]*gitlab[^/]+)\\/(.+?)\\/?$/i)!\n      return fetchSourceFactsFromGitLab(base, slug)\n\n    /* Everything else */\n    default:\n      return NEVER\n  }\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { NEVER, Observable, Subject, defer, of } from \"rxjs\"\nimport {\n  catchError,\n  filter,\n  map,\n  shareReplay,\n  tap\n} from \"rxjs/operators\"\n\nimport { setSourceFacts, setSourceState } from \"~/actions\"\nimport { renderSourceFacts } from \"~/templates\"\n\nimport { Component } from \"../../_\"\nimport { SourceFacts, fetchSourceFacts } from \"../facts\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Repository information\n */\nexport interface Source {\n  facts: SourceFacts                   /* Repository facts */\n}\n\n/* ----------------------------------------------------------------------------\n * Data\n * ------------------------------------------------------------------------- */\n\n/**\n * Repository information observable\n */\nlet fetch$: Observable<Source>\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch repository information\n *\n * This function tries to read the repository facts from session storage, and\n * if unsuccessful, fetches them from the underlying provider.\n *\n * @param el - Repository information element\n *\n * @returns Repository information observable\n */\nexport function watchSource(\n  el: HTMLAnchorElement\n): Observable<Source> {\n  return fetch$ ||= defer(() => {\n    const data = sessionStorage.getItem(__prefix(\"__source\"))\n    if (data) {\n      return of<SourceFacts>(JSON.parse(data))\n    } else {\n      const value$ = fetchSourceFacts(el.href)\n      value$.subscribe(value => {\n        try {\n          sessionStorage.setItem(__prefix(\"__source\"), JSON.stringify(value))\n        } catch (err) {\n          /* Uncritical, just swallow */\n        }\n      })\n\n      /* Return value */\n      return value$\n    }\n  })\n    .pipe(\n      catchError(() => NEVER),\n      filter(facts => Object.keys(facts).length > 0),\n      map(facts => ({ facts })),\n      shareReplay(1)\n    )\n}\n\n/**\n * Mount repository information\n *\n * @param el - Repository information element\n *\n * @returns Repository information component observable\n */\nexport function mountSource(\n  el: HTMLAnchorElement\n): Observable<Component<Source>> {\n  const internal$ = new Subject<Source>()\n  internal$.subscribe(({ facts }) => {\n    setSourceFacts(el, renderSourceFacts(facts))\n    setSourceState(el, \"done\")\n  })\n\n  /* Create and return component */\n  return watchSource(el)\n    .pipe(\n      tap(internal$),\n      map(state => ({ ref: el, ...state }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Observable, Subject, animationFrameScheduler } from \"rxjs\"\nimport {\n  distinctUntilKeyChanged,\n  map,\n  observeOn,\n  switchMap,\n  tap\n} from \"rxjs/operators\"\n\nimport { resetTabsState, setTabsState } from \"~/actions\"\nimport {\n  Viewport,\n  watchElementSize,\n  watchViewportAt\n} from \"~/browser\"\n\nimport { Component } from \"../_\"\nimport { Header } from \"../header\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Navigation tabs\n */\nexport interface Tabs {\n  hidden: boolean                      /* User scrolled past tabs */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch navigation tabs\n *\n * @param el - Navigation tabs element\n * @param options - Options\n *\n * @returns Navigation tabs observable\n */\nexport function watchTabs(\n  el: HTMLElement, { viewport$, header$ }: WatchOptions\n): Observable<Tabs> {\n  return watchElementSize(document.body)\n    .pipe(\n      switchMap(() => watchViewportAt(el, { header$, viewport$ })),\n      map(({ offset: { y } }) => {\n        return {\n          hidden: y >= 10\n        }\n      }),\n      distinctUntilKeyChanged(\"hidden\")\n    )\n}\n\n/**\n * Mount navigation tabs\n *\n * This function hides the navigation tabs when scrolling past the threshold\n * and makes them reappear in a nice CSS animation when scrolling back up.\n *\n * @param el - Navigation tabs element\n * @param options - Options\n *\n * @returns Navigation tabs component observable\n */\nexport function mountTabs(\n  el: HTMLElement, options: MountOptions\n): Observable<Component<Tabs>> {\n  const internal$ = new Subject<Tabs>()\n  internal$\n    .pipe(\n      observeOn(animationFrameScheduler)\n    )\n      .subscribe({\n\n        /* Update state */\n        next({ hidden }) {\n          if (hidden)\n            setTabsState(el, \"hidden\")\n          else\n            resetTabsState(el)\n        },\n\n        /* Reset on complete */\n        complete() {\n          resetTabsState(el)\n        }\n      })\n\n  /* Create and return component */\n  return watchTabs(el, options)\n    .pipe(\n      tap(internal$),\n      map(state => ({ ref: el, ...state }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  animationFrameScheduler,\n  combineLatest\n} from \"rxjs\"\nimport {\n  bufferCount,\n  distinctUntilChanged,\n  distinctUntilKeyChanged,\n  map,\n  observeOn,\n  scan,\n  startWith,\n  switchMap,\n  tap\n} from \"rxjs/operators\"\n\nimport {\n  resetAnchorActive,\n  resetAnchorState,\n  setAnchorActive,\n  setAnchorState\n} from \"~/actions\"\nimport {\n  Viewport,\n  getElement,\n  getElements,\n  watchElementSize\n} from \"~/browser\"\n\nimport { Component } from \"../_\"\nimport { Header } from \"../header\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Table of contents\n */\nexport interface TableOfContents {\n  prev: HTMLAnchorElement[][]          /* Anchors (previous) */\n  next: HTMLAnchorElement[][]          /* Anchors (next) */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch table of contents\n *\n * This is effectively a scroll spy implementation which will account for the\n * fixed header and automatically re-calculate anchor offsets when the viewport\n * is resized. The returned observable will only emit if the table of contents\n * needs to be repainted.\n *\n * This implementation tracks an anchor element's entire path starting from its\n * level up to the top-most anchor element, e.g. `[h3, h2, h1]`. Although the\n * Material theme currently doesn't make use of this information, it enables\n * the styling of the entire hierarchy through customization.\n *\n * Note that the current anchor is the last item of the `prev` anchor list.\n *\n * @param anchors - Anchor elements\n * @param options - Options\n *\n * @returns Table of contents observable\n */\nexport function watchTableOfContents(\n  anchors: HTMLAnchorElement[], { viewport$, header$ }: WatchOptions\n): Observable<TableOfContents> {\n  const table = new Map<HTMLAnchorElement, HTMLElement>()\n  for (const anchor of anchors) {\n    const id = decodeURIComponent(anchor.hash.substring(1))\n    const target = getElement(`[id=\"${id}\"]`)\n    if (typeof target !== \"undefined\")\n      table.set(anchor, target)\n  }\n\n  /* Compute necessary adjustment for header */\n  const adjust$ = header$\n    .pipe(\n      map(header => 24 + header.height)\n    )\n\n  /* Compute partition of previous and next anchors */\n  const partition$ = watchElementSize(document.body)\n    .pipe(\n      distinctUntilKeyChanged(\"height\"),\n\n      /* Build index to map anchor paths to vertical offsets */\n      map(() => {\n        let path: HTMLAnchorElement[] = []\n        return [...table].reduce((index, [anchor, target]) => {\n          while (path.length) {\n            const last = table.get(path[path.length - 1])!\n            if (last.tagName >= target.tagName) {\n              path.pop()\n            } else {\n              break\n            }\n          }\n\n          /* If the current anchor is hidden, continue with its parent */\n          let offset = target.offsetTop\n          while (!offset && target.parentElement) {\n            target = target.parentElement\n            offset = target.offsetTop\n          }\n\n          /* Map reversed anchor path to vertical offset */\n          return index.set(\n            [...path = [...path, anchor]].reverse(),\n            offset\n          )\n        }, new Map<HTMLAnchorElement[], number>())\n      }),\n\n      /* Sort index by vertical offset (see https://bit.ly/30z6QSO) */\n      map(index => new Map([...index].sort(([, a], [, b]) => a - b))),\n\n      /* Re-compute partition when viewport offset changes */\n      switchMap(index => combineLatest([adjust$, viewport$])\n        .pipe(\n          scan(([prev, next], [adjust, { offset: { y } }]) => {\n\n            /* Look forward */\n            while (next.length) {\n              const [, offset] = next[0]\n              if (offset - adjust < y) {\n                prev = [...prev, next.shift()!]\n              } else {\n                break\n              }\n            }\n\n            /* Look backward */\n            while (prev.length) {\n              const [, offset] = prev[prev.length - 1]\n              if (offset - adjust >= y) {\n                next = [prev.pop()!, ...next]\n              } else {\n                break\n              }\n            }\n\n            /* Return partition */\n            return [prev, next]\n          }, [[], [...index]]),\n          distinctUntilChanged((a, b) => (\n            a[0] === b[0] &&\n            a[1] === b[1]\n          ))\n        )\n      )\n    )\n\n  /* Compute and return anchor list migrations */\n  return partition$\n    .pipe(\n      map(([prev, next]) => ({\n        prev: prev.map(([path]) => path),\n        next: next.map(([path]) => path)\n      })),\n\n      /* Extract anchor list migrations */\n      startWith({ prev: [], next: [] }),\n      bufferCount(2, 1),\n      map(([a, b]) => {\n\n        /* Moving down */\n        if (a.prev.length < b.prev.length) {\n          return {\n            prev: b.prev.slice(Math.max(0, a.prev.length - 1), b.prev.length),\n            next: []\n          }\n\n        /* Moving up */\n        } else {\n          return {\n            prev: b.prev.slice(-1),\n            next: b.next.slice(0, b.next.length - a.next.length)\n          }\n        }\n      })\n    )\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Mount table of contents\n *\n * @param el - Anchor list element\n * @param options - Options\n *\n * @returns Table of contents component observable\n */\nexport function mountTableOfContents(\n  el: HTMLElement, options: MountOptions\n): Observable<Component<TableOfContents>> {\n  const internal$ = new Subject<TableOfContents>()\n  internal$\n    .pipe(\n      observeOn(animationFrameScheduler),\n    )\n      .subscribe(({ prev, next }) => {\n\n        /* Look forward */\n        for (const [anchor] of next) {\n          resetAnchorActive(anchor)\n          resetAnchorState(anchor)\n        }\n\n        /* Look backward */\n        for (const [index, [anchor]] of prev.entries()) {\n          setAnchorActive(anchor, index === prev.length - 1)\n          setAnchorState(anchor, \"blur\")\n        }\n      })\n\n  /* Create and return component */\n  const anchors = getElements<HTMLAnchorElement>(\"[href^=\\\\#]\", el)\n  return watchTableOfContents(anchors, options)\n    .pipe(\n      tap(internal$),\n      map(state => ({ ref: el, ...state }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  Subject,\n  animationFrameScheduler,\n  combineLatest\n} from \"rxjs\"\nimport {\n  bufferCount,\n  distinctUntilChanged,\n  distinctUntilKeyChanged,\n  map,\n  observeOn,\n  tap,\n  withLatestFrom\n} from \"rxjs/operators\"\n\nimport {\n  resetBackToTopOffset,\n  resetBackToTopState,\n  setBackToTopOffset,\n  setBackToTopState\n} from \"~/actions\"\nimport { Viewport, setElementFocus } from \"~/browser\"\n\nimport { Component } from \"../_\"\nimport { Header } from \"../header\"\nimport { Main } from \"../main\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Back-to-top button\n */\nexport interface BackToTop {\n  hidden: boolean                      /* User scrolled up */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch options\n */\ninterface WatchOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n  main$: Observable<Main>              /* Main area observable */\n}\n\n/**\n * Mount options\n */\ninterface MountOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  header$: Observable<Header>          /* Header observable */\n  main$: Observable<Main>              /* Main area observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Watch back-to-top\n *\n * @param _el - Back-to-top element\n * @param options - Options\n *\n * @returns Back-to-top observable\n */\nexport function watchBackToTop(\n  _el: HTMLElement, { viewport$, main$ }: WatchOptions\n): Observable<BackToTop> {\n\n  /* Compute direction */\n  const direction$ = viewport$\n    .pipe(\n      map(({ offset: { y } }) => y),\n      bufferCount(2, 1),\n      map(([a, b]) => a > b && b),\n      distinctUntilChanged()\n    )\n\n  /* Compute whether button should be hidden */\n  const hidden$ = main$\n    .pipe(\n      distinctUntilKeyChanged(\"active\")\n    )\n\n  /* Compute threshold for hiding */\n  return combineLatest([hidden$, direction$])\n    .pipe(\n      map(([{ active }, direction]) => ({\n        hidden: !(active && direction)\n      })),\n      distinctUntilChanged((a, b) => (\n        a.hidden === b.hidden\n      ))\n    )\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Mount back-to-top\n *\n * @param el - Back-to-top element\n * @param options - Options\n *\n * @returns Back-to-top component observable\n */\nexport function mountBackToTop(\n  el: HTMLElement, { viewport$, header$, main$ }: MountOptions\n): Observable<Component<BackToTop>> {\n  const internal$ = new Subject<BackToTop>()\n  internal$\n    .pipe(\n      observeOn(animationFrameScheduler),\n      withLatestFrom(header$\n        .pipe(\n          distinctUntilKeyChanged(\"height\")\n        )\n      )\n    )\n      .subscribe({\n\n        /* Update state */\n        next([{ hidden }, { height }]) {\n          setBackToTopOffset(el, height + 16)\n          if (hidden) {\n            setBackToTopState(el, \"hidden\")\n            setElementFocus(el, false)\n          } else {\n            resetBackToTopState(el)\n          }\n        },\n\n        /* Reset on complete */\n        complete() {\n          resetBackToTopOffset(el)\n          resetBackToTopState(el)\n        }\n      })\n\n  /* Create and return component */\n  return watchBackToTop(el, { viewport$, header$, main$ })\n    .pipe(\n      tap(internal$),\n      map(state => ({ ref: el, ...state }))\n    )\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Observable, fromEvent, of } from \"rxjs\"\nimport {\n  mapTo,\n  mergeMap,\n  switchMap,\n  takeWhile,\n  tap,\n  withLatestFrom\n} from \"rxjs/operators\"\n\nimport { getElements } from \"~/browser\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Patch options\n */\ninterface PatchOptions {\n  document$: Observable<Document>      /* Document observable */\n  tablet$: Observable<boolean>         /* Tablet breakpoint observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Patch indeterminate checkboxes\n *\n * This function replaces the indeterminate \"pseudo state\" with the actual\n * indeterminate state, which is used to keep navigation always expanded.\n *\n * @param options - Options\n */\nexport function patchIndeterminate(\n  { document$, tablet$ }: PatchOptions\n): void {\n  document$\n    .pipe(\n      switchMap(() => of(...getElements<HTMLInputElement>(\n        \"[data-md-state=indeterminate]\"\n      ))),\n      tap(el => {\n        el.indeterminate = true\n        el.checked = false\n      }),\n      mergeMap(el => fromEvent(el, \"change\")\n        .pipe(\n          takeWhile(() => el.hasAttribute(\"data-md-state\")),\n          mapTo(el)\n        )\n      ),\n      withLatestFrom(tablet$)\n    )\n      .subscribe(([el, tablet]) => {\n        el.removeAttribute(\"data-md-state\")\n        if (tablet)\n          el.checked = false\n      })\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { Observable, fromEvent, of } from \"rxjs\"\nimport {\n  filter,\n  mapTo,\n  mergeMap,\n  switchMap,\n  tap\n} from \"rxjs/operators\"\n\nimport { getElements } from \"~/browser\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Patch options\n */\ninterface PatchOptions {\n  document$: Observable<Document>      /* Document observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Check whether the given device is an Apple device\n *\n * @returns Test result\n */\nfunction isAppleDevice(): boolean {\n  return /(iPad|iPhone|iPod)/.test(navigator.userAgent)\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Patch all elements with `data-md-scrollfix` attributes\n *\n * This is a year-old patch which ensures that overflow scrolling works at the\n * top and bottom of containers on iOS by ensuring a `1px` scroll offset upon\n * the start of a touch event.\n *\n * @see https://bit.ly/2SCtAOO - Original source\n *\n * @param options - Options\n */\nexport function patchScrollfix(\n  { document$ }: PatchOptions\n): void {\n  document$\n    .pipe(\n      switchMap(() => of(...getElements(\"[data-md-scrollfix]\"))),\n      tap(el => el.removeAttribute(\"data-md-scrollfix\")),\n      filter(isAppleDevice),\n      mergeMap(el => fromEvent(el, \"touchstart\")\n        .pipe(\n          mapTo(el)\n        )\n      )\n    )\n      .subscribe(el => {\n        const top = el.scrollTop\n\n        /* We're at the top of the container */\n        if (top === 0) {\n          el.scrollTop = 1\n\n        /* We're at the bottom of the container */\n        } else if (top + el.offsetHeight === el.scrollHeight) {\n          el.scrollTop = top - 1\n        }\n      })\n}\n", "/*\n * Copyright (c) 2016-2021 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  Observable,\n  animationFrameScheduler,\n  combineLatest,\n  of\n} from \"rxjs\"\nimport {\n  delay,\n  map,\n  observeOn,\n  switchMap,\n  withLatestFrom\n} from \"rxjs/operators\"\n\nimport { resetScrollLock, setScrollLock } from \"~/actions\"\nimport { Viewport, watchToggle } from \"~/browser\"\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Patch options\n */\ninterface PatchOptions {\n  viewport$: Observable<Viewport>      /* Viewport observable */\n  tablet$: Observable<boolean>         /* Tablet breakpoint observable */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Patch the document body to lock when search is open\n *\n * For mobile and tablet viewports, the search is rendered full screen, which\n * leads to scroll leaking when at the top or bottom of the search result. This\n * function locks the body when the search is in full screen mode, and restores\n * the scroll position when leaving.\n *\n * @param options - Options\n */\nexport function patchScrolllock(\n  { viewport$, tablet$ }: PatchOptions\n): void {\n  combineLatest([watchToggle(\"search\"), tablet$])\n    .pipe(\n      map(([active, tablet]) => active && !tablet),\n      switchMap(active => of(active)\n        .pipe(\n          delay(active ? 400 : 100),\n          observeOn(animationFrameScheduler)\n        )\n      ),\n      withLatestFrom(viewport$)\n    )\n      .subscribe(([active, { offset: { y }}]) => {\n        if (active)\n          setScrollLock(document.body, y)\n        else\n          resetScrollLock(document.body)\n      })\n}\n"],
-  "mappings": "4iCAAA,oBAAC,UAAU,EAAQ,EAAS,CAC1B,MAAO,KAAY,UAAY,MAAO,KAAW,YAAc,IAC/D,MAAO,SAAW,YAAc,OAAO,IAAM,OAAO,GACnD,MACD,GAAO,UAAY,CAAE,aASrB,WAAmC,EAAO,CACxC,GAAI,GAAmB,GACnB,EAA0B,GAC1B,EAAiC,KAEjC,EAAsB,CACxB,KAAM,GACN,OAAQ,GACR,IAAK,GACL,IAAK,GACL,MAAO,GACP,SAAU,GACV,OAAQ,GACR,KAAM,GACN,MAAO,GACP,KAAM,GACN,KAAM,GACN,SAAU,GACV,iBAAkB,IAQpB,WAA4B,EAAI,CAC9B,MACE,MACA,IAAO,UACP,EAAG,WAAa,QAChB,EAAG,WAAa,QAChB,aAAe,IACf,YAAc,GAAG,WAcrB,WAAuC,EAAI,CACzC,GAAI,IAAO,EAAG,KACV,GAAU,EAAG,QAUjB,MARI,QAAY,SAAW,EAAoB,KAAS,CAAC,EAAG,UAIxD,KAAY,YAAc,CAAC,EAAG,UAI9B,EAAG,mBAYT,WAA8B,EAAI,CAChC,AAAI,EAAG,UAAU,SAAS,kBAG1B,GAAG,UAAU,IAAI,iBACjB,EAAG,aAAa,2BAA4B,KAQ9C,WAAiC,EAAI,CACnC,AAAI,CAAC,EAAG,aAAa,6BAGrB,GAAG,UAAU,OAAO,iBACpB,EAAG,gBAAgB,6BAWrB,WAAmB,EAAG,CACpB,AAAI,EAAE,SAAW,EAAE,QAAU,EAAE,SAI3B,GAAmB,EAAM,gBAC3B,EAAqB,EAAM,eAG7B,EAAmB,IAWrB,WAAuB,EAAG,CACxB,EAAmB,GAUrB,WAAiB,EAAG,CAElB,AAAI,CAAC,EAAmB,EAAE,SAItB,IAAoB,EAA8B,EAAE,UACtD,EAAqB,EAAE,QAQ3B,WAAgB,EAAG,CACjB,AAAI,CAAC,EAAmB,EAAE,SAKxB,GAAE,OAAO,UAAU,SAAS,kBAC5B,EAAE,OAAO,aAAa,8BAMtB,GAA0B,GAC1B,OAAO,aAAa,GACpB,EAAiC,OAAO,WAAW,UAAW,CAC5D,EAA0B,IACzB,KACH,EAAwB,EAAE,SAS9B,WAA4B,EAAG,CAC7B,AAAI,SAAS,kBAAoB,UAK3B,IACF,GAAmB,IAErB,KAUJ,YAA0C,CACxC,SAAS,iBAAiB,YAAa,GACvC,SAAS,iBAAiB,YAAa,GACvC,SAAS,iBAAiB,UAAW,GACrC,SAAS,iBAAiB,cAAe,GACzC,SAAS,iBAAiB,cAAe,GACzC,SAAS,iBAAiB,YAAa,GACvC,SAAS,iBAAiB,YAAa,GACvC,SAAS,iBAAiB,aAAc,GACxC,SAAS,iBAAiB,WAAY,GAGxC,YAA6C,CAC3C,SAAS,oBAAoB,YAAa,GAC1C,SAAS,oBAAoB,YAAa,GAC1C,SAAS,oBAAoB,UAAW,GACxC,SAAS,oBAAoB,cAAe,GAC5C,SAAS,oBAAoB,cAAe,GAC5C,SAAS,oBAAoB,YAAa,GAC1C,SAAS,oBAAoB,YAAa,GAC1C,SAAS,oBAAoB,aAAc,GAC3C,SAAS,oBAAoB,WAAY,GAU3C,WAA8B,EAAG,CAG/B,AAAI,EAAE,OAAO,UAAY,EAAE,OAAO,SAAS,gBAAkB,QAI7D,GAAmB,GACnB,KAMF,SAAS,iBAAiB,UAAW,EAAW,IAChD,SAAS,iBAAiB,YAAa,EAAe,IACtD,SAAS,iBAAiB,cAAe,EAAe,IACxD,SAAS,iBAAiB,aAAc,EAAe,IACvD,SAAS,iBAAiB,mBAAoB,EAAoB,IAElE,IAMA,EAAM,iBAAiB,QAAS,EAAS,IACzC,EAAM,iBAAiB,OAAQ,EAAQ,IAOvC,AAAI,EAAM,WAAa,KAAK,wBAA0B,EAAM,KAI1D,EAAM,KAAK,aAAa,wBAAyB,IACxC,EAAM,WAAa,KAAK,eACjC,UAAS,gBAAgB,UAAU,IAAI,oBACvC,SAAS,gBAAgB,aAAa,wBAAyB,KAOnE,GAAI,MAAO,SAAW,aAAe,MAAO,WAAa,YAAa,CAIpE,OAAO,0BAA4B,EAInC,GAAI,GAEJ,GAAI,CACF,EAAQ,GAAI,aAAY,sCACjB,EAAP,CAEA,EAAQ,SAAS,YAAY,eAC7B,EAAM,gBAAgB,+BAAgC,GAAO,GAAO,IAGtE,OAAO,cAAc,GAGvB,AAAI,MAAO,WAAa,aAGtB,EAA0B,cCpT9B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,gFAeA,GAAI,IACA,GACA,GACA,GACA,GACA,GACA,GACA,GACA,GACA,GACA,GACA,GACA,GACA,GACA,GACA,GACA,GACA,GACA,GACA,GACA,GACA,GACA,GACA,GACJ,AAAC,UAAU,EAAS,CAChB,GAAI,GAAO,MAAO,SAAW,SAAW,OAAS,MAAO,OAAS,SAAW,KAAO,MAAO,OAAS,SAAW,KAAO,GACrH,AAAI,MAAO,SAAW,YAAc,OAAO,IACvC,OAAO,QAAS,CAAC,WAAY,SAAU,EAAS,CAAE,EAAQ,EAAe,EAAM,EAAe,OAE7F,AAAI,MAAO,KAAW,UAAY,MAAO,IAAO,SAAY,SAC7D,EAAQ,EAAe,EAAM,EAAe,GAAO,WAGnD,EAAQ,EAAe,IAE3B,WAAwB,EAAS,EAAU,CACvC,MAAI,KAAY,GACZ,CAAI,MAAO,QAAO,QAAW,WACzB,OAAO,eAAe,EAAS,aAAc,CAAE,MAAO,KAGtD,EAAQ,WAAa,IAGtB,SAAU,EAAI,EAAG,CAAE,MAAO,GAAQ,GAAM,EAAW,EAAS,EAAI,GAAK,MAGnF,SAAU,EAAU,CACjB,GAAI,GAAgB,OAAO,gBACtB,CAAE,UAAW,aAAgB,QAAS,SAAU,EAAG,EAAG,CAAE,EAAE,UAAY,IACvE,SAAU,EAAG,EAAG,CAAE,OAAS,KAAK,GAAG,AAAI,OAAO,UAAU,eAAe,KAAK,EAAG,IAAI,GAAE,GAAK,EAAE,KAEhG,GAAY,SAAU,EAAG,EAAG,CACxB,GAAI,MAAO,IAAM,YAAc,IAAM,KACjC,KAAM,IAAI,WAAU,uBAAyB,OAAO,GAAK,iCAC7D,EAAc,EAAG,GACjB,YAAc,CAAE,KAAK,YAAc,EACnC,EAAE,UAAY,IAAM,KAAO,OAAO,OAAO,GAAM,GAAG,UAAY,EAAE,UAAW,GAAI,KAGnF,GAAW,OAAO,QAAU,SAAU,EAAG,CACrC,OAAS,GAAG,EAAI,EAAG,EAAI,UAAU,OAAQ,EAAI,EAAG,IAAK,CACjD,EAAI,UAAU,GACd,OAAS,KAAK,GAAG,AAAI,OAAO,UAAU,eAAe,KAAK,EAAG,IAAI,GAAE,GAAK,EAAE,IAE9E,MAAO,IAGX,GAAS,SAAU,EAAG,EAAG,CACrB,GAAI,GAAI,GACR,OAAS,KAAK,GAAG,AAAI,OAAO,UAAU,eAAe,KAAK,EAAG,IAAM,EAAE,QAAQ,GAAK,GAC9E,GAAE,GAAK,EAAE,IACb,GAAI,GAAK,MAAQ,MAAO,QAAO,uBAA0B,WACrD,OAAS,GAAI,EAAG,EAAI,OAAO,sBAAsB,GAAI,EAAI,EAAE,OAAQ,IAC/D,AAAI,EAAE,QAAQ,EAAE,IAAM,GAAK,OAAO,UAAU,qBAAqB,KAAK,EAAG,EAAE,KACvE,GAAE,EAAE,IAAM,EAAE,EAAE,KAE1B,MAAO,IAGX,GAAa,SAAU,EAAY,EAAQ,EAAK,EAAM,CAClD,GAAI,GAAI,UAAU,OAAQ,EAAI,EAAI,EAAI,EAAS,IAAS,KAAO,EAAO,OAAO,yBAAyB,EAAQ,GAAO,EAAM,EAC3H,GAAI,MAAO,UAAY,UAAY,MAAO,SAAQ,UAAa,WAAY,EAAI,QAAQ,SAAS,EAAY,EAAQ,EAAK,OACpH,QAAS,GAAI,EAAW,OAAS,EAAG,GAAK,EAAG,IAAK,AAAI,GAAI,EAAW,KAAI,GAAK,GAAI,EAAI,EAAE,GAAK,EAAI,EAAI,EAAE,EAAQ,EAAK,GAAK,EAAE,EAAQ,KAAS,GAChJ,MAAO,GAAI,GAAK,GAAK,OAAO,eAAe,EAAQ,EAAK,GAAI,GAGhE,GAAU,SAAU,EAAY,EAAW,CACvC,MAAO,UAAU,EAAQ,EAAK,CAAE,EAAU,EAAQ,EAAK,KAG3D,GAAa,SAAU,EAAa,EAAe,CAC/C,GAAI,MAAO,UAAY,UAAY,MAAO,SAAQ,UAAa,WAAY,MAAO,SAAQ,SAAS,EAAa,IAGpH,GAAY,SAAU,EAAS,EAAY,EAAG,EAAW,CACrD,WAAe,EAAO,CAAE,MAAO,aAAiB,GAAI,EAAQ,GAAI,GAAE,SAAU,EAAS,CAAE,EAAQ,KAC/F,MAAO,IAAK,IAAM,GAAI,UAAU,SAAU,EAAS,EAAQ,CACvD,WAAmB,EAAO,CAAE,GAAI,CAAE,EAAK,EAAU,KAAK,UAAkB,EAAP,CAAY,EAAO,IACpF,WAAkB,EAAO,CAAE,GAAI,CAAE,EAAK,EAAU,MAAS,UAAkB,EAAP,CAAY,EAAO,IACvF,WAAc,EAAQ,CAAE,EAAO,KAAO,EAAQ,EAAO,OAAS,EAAM,EAAO,OAAO,KAAK,EAAW,GAClG,EAAM,GAAY,EAAU,MAAM,EAAS,GAAc,KAAK,WAItE,GAAc,SAAU,EAAS,EAAM,CACnC,GAAI,GAAI,CAAE,MAAO,EAAG,KAAM,UAAW,CAAE,GAAI,EAAE,GAAK,EAAG,KAAM,GAAE,GAAI,MAAO,GAAE,IAAO,KAAM,GAAI,IAAK,IAAM,EAAG,EAAG,EAAG,EAC/G,MAAO,GAAI,CAAE,KAAM,EAAK,GAAI,MAAS,EAAK,GAAI,OAAU,EAAK,IAAM,MAAO,SAAW,YAAe,GAAE,OAAO,UAAY,UAAW,CAAE,MAAO,QAAU,EACvJ,WAAc,EAAG,CAAE,MAAO,UAAU,EAAG,CAAE,MAAO,GAAK,CAAC,EAAG,KACzD,WAAc,EAAI,CACd,GAAI,EAAG,KAAM,IAAI,WAAU,mCAC3B,KAAO,GAAG,GAAI,CACV,GAAI,EAAI,EAAG,GAAM,GAAI,EAAG,GAAK,EAAI,EAAE,OAAY,EAAG,GAAK,EAAE,OAAc,IAAI,EAAE,SAAc,EAAE,KAAK,GAAI,GAAK,EAAE,OAAS,CAAE,GAAI,EAAE,KAAK,EAAG,EAAG,KAAK,KAAM,MAAO,GAE3J,OADI,EAAI,EAAG,GAAG,GAAK,CAAC,EAAG,GAAK,EAAG,EAAE,QACzB,EAAG,QACF,OAAQ,GAAG,EAAI,EAAI,UACnB,GAAG,SAAE,QAAgB,CAAE,MAAO,EAAG,GAAI,KAAM,QAC3C,GAAG,EAAE,QAAS,EAAI,EAAG,GAAI,EAAK,CAAC,GAAI,aACnC,GAAG,EAAK,EAAE,IAAI,MAAO,EAAE,KAAK,MAAO,iBAEpC,GAAM,EAAI,EAAE,KAAM,IAAI,EAAE,OAAS,GAAK,EAAE,EAAE,OAAS,KAAQ,GAAG,KAAO,GAAK,EAAG,KAAO,GAAI,CAAE,EAAI,EAAG,SACjG,GAAI,EAAG,KAAO,GAAM,EAAC,GAAM,EAAG,GAAK,EAAE,IAAM,EAAG,GAAK,EAAE,IAAM,CAAE,EAAE,MAAQ,EAAG,GAAI,MAC9E,GAAI,EAAG,KAAO,GAAK,EAAE,MAAQ,EAAE,GAAI,CAAE,EAAE,MAAQ,EAAE,GAAI,EAAI,EAAI,MAC7D,GAAI,GAAK,EAAE,MAAQ,EAAE,GAAI,CAAE,EAAE,MAAQ,EAAE,GAAI,EAAE,IAAI,KAAK,GAAK,MAC3D,AAAI,EAAE,IAAI,EAAE,IAAI,MAChB,EAAE,KAAK,MAAO,SAEtB,EAAK,EAAK,KAAK,EAAS,SACnB,EAAP,CAAY,EAAK,CAAC,EAAG,GAAI,EAAI,SAAK,CAAU,EAAI,EAAI,EACtD,GAAI,EAAG,GAAK,EAAG,KAAM,GAAG,GAAI,MAAO,CAAE,MAAO,EAAG,GAAK,EAAG,GAAK,OAAQ,KAAM,MAIlF,GAAe,SAAS,EAAG,EAAG,CAC1B,OAAS,KAAK,GAAG,AAAI,IAAM,WAAa,CAAC,OAAO,UAAU,eAAe,KAAK,EAAG,IAAI,GAAgB,EAAG,EAAG,IAG/G,GAAkB,OAAO,OAAU,SAAS,EAAG,EAAG,EAAG,EAAI,CACrD,AAAI,IAAO,QAAW,GAAK,GAC3B,OAAO,eAAe,EAAG,EAAI,CAAE,WAAY,GAAM,IAAK,UAAW,CAAE,MAAO,GAAE,OAC1E,SAAS,EAAG,EAAG,EAAG,EAAI,CACxB,AAAI,IAAO,QAAW,GAAK,GAC3B,EAAE,GAAM,EAAE,IAGd,GAAW,SAAU,EAAG,CACpB,GAAI,GAAI,MAAO,SAAW,YAAc,OAAO,SAAU,EAAI,GAAK,EAAE,GAAI,EAAI,EAC5E,GAAI,EAAG,MAAO,GAAE,KAAK,GACrB,GAAI,GAAK,MAAO,GAAE,QAAW,SAAU,MAAO,CAC1C,KAAM,UAAY,CACd,MAAI,IAAK,GAAK,EAAE,QAAQ,GAAI,QACrB,CAAE,MAAO,GAAK,EAAE,KAAM,KAAM,CAAC,KAG5C,KAAM,IAAI,WAAU,EAAI,0BAA4B,oCAGxD,GAAS,SAAU,EAAG,EAAG,CACrB,GAAI,GAAI,MAAO,SAAW,YAAc,EAAE,OAAO,UACjD,GAAI,CAAC,EAAG,MAAO,GACf,GAAI,GAAI,EAAE,KAAK,GAAI,EAAG,EAAK,GAAI,EAC/B,GAAI,CACA,KAAQ,KAAM,QAAU,KAAM,IAAM,CAAE,GAAI,EAAE,QAAQ,MAAM,EAAG,KAAK,EAAE,aAEjE,EAAP,CAAgB,EAAI,CAAE,MAAO,UAC7B,CACI,GAAI,CACA,AAAI,GAAK,CAAC,EAAE,MAAS,GAAI,EAAE,SAAY,EAAE,KAAK,UAElD,CAAU,GAAI,EAAG,KAAM,GAAE,OAE7B,MAAO,IAIX,GAAW,UAAY,CACnB,OAAS,GAAK,GAAI,EAAI,EAAG,EAAI,UAAU,OAAQ,IAC3C,EAAK,EAAG,OAAO,GAAO,UAAU,KACpC,MAAO,IAIX,GAAiB,UAAY,CACzB,OAAS,GAAI,EAAG,EAAI,EAAG,EAAK,UAAU,OAAQ,EAAI,EAAI,IAAK,GAAK,UAAU,GAAG,OAC7E,OAAS,GAAI,MAAM,GAAI,EAAI,EAAG,EAAI,EAAG,EAAI,EAAI,IACzC,OAAS,GAAI,UAAU,GAAI,EAAI,EAAG,EAAK,EAAE,OAAQ,EAAI,EAAI,IAAK,IAC1D,EAAE,GAAK,EAAE,GACjB,MAAO,IAGX,GAAgB,SAAU,EAAI,EAAM,CAChC,OAAS,GAAI,EAAG,EAAK,EAAK,OAAQ,EAAI,EAAG,OAAQ,EAAI,EAAI,IAAK,IAC1D,EAAG,GAAK,EAAK,GACjB,MAAO,IAGX,GAAU,SAAU,EAAG,CACnB,MAAO,gBAAgB,IAAW,MAAK,EAAI,EAAG,MAAQ,GAAI,IAAQ,IAGtE,GAAmB,SAAU,EAAS,EAAY,EAAW,CACzD,GAAI,CAAC,OAAO,cAAe,KAAM,IAAI,WAAU,wCAC/C,GAAI,GAAI,EAAU,MAAM,EAAS,GAAc,IAAK,EAAG,EAAI,GAC3D,MAAO,GAAI,GAAI,EAAK,QAAS,EAAK,SAAU,EAAK,UAAW,EAAE,OAAO,eAAiB,UAAY,CAAE,MAAO,OAAS,EACpH,WAAc,EAAG,CAAE,AAAI,EAAE,IAAI,GAAE,GAAK,SAAU,EAAG,CAAE,MAAO,IAAI,SAAQ,SAAU,EAAG,EAAG,CAAE,EAAE,KAAK,CAAC,EAAG,EAAG,EAAG,IAAM,GAAK,EAAO,EAAG,OAC9H,WAAgB,EAAG,EAAG,CAAE,GAAI,CAAE,EAAK,EAAE,GAAG,UAAc,EAAP,CAAY,EAAO,EAAE,GAAG,GAAI,IAC3E,WAAc,EAAG,CAAE,EAAE,gBAAiB,IAAU,QAAQ,QAAQ,EAAE,MAAM,GAAG,KAAK,EAAS,GAAU,EAAO,EAAE,GAAG,GAAI,GACnH,WAAiB,EAAO,CAAE,EAAO,OAAQ,GACzC,WAAgB,EAAO,CAAE,EAAO,QAAS,GACzC,WAAgB,EAAG,EAAG,CAAE,AAAI,EAAE,GAAI,EAAE,QAAS,EAAE,QAAQ,EAAO,EAAE,GAAG,GAAI,EAAE,GAAG,MAGhF,GAAmB,SAAU,EAAG,CAC5B,GAAI,GAAG,EACP,MAAO,GAAI,GAAI,EAAK,QAAS,EAAK,QAAS,SAAU,EAAG,CAAE,KAAM,KAAO,EAAK,UAAW,EAAE,OAAO,UAAY,UAAY,CAAE,MAAO,OAAS,EAC1I,WAAc,EAAG,EAAG,CAAE,EAAE,GAAK,EAAE,GAAK,SAAU,EAAG,CAAE,MAAQ,GAAI,CAAC,GAAK,CAAE,MAAO,GAAQ,EAAE,GAAG,IAAK,KAAM,IAAM,UAAa,EAAI,EAAE,GAAK,GAAO,IAG/I,GAAgB,SAAU,EAAG,CACzB,GAAI,CAAC,OAAO,cAAe,KAAM,IAAI,WAAU,wCAC/C,GAAI,GAAI,EAAE,OAAO,eAAgB,EACjC,MAAO,GAAI,EAAE,KAAK,GAAM,GAAI,MAAO,KAAa,WAAa,GAAS,GAAK,EAAE,OAAO,YAAa,EAAI,GAAI,EAAK,QAAS,EAAK,SAAU,EAAK,UAAW,EAAE,OAAO,eAAiB,UAAY,CAAE,MAAO,OAAS,GAC9M,WAAc,EAAG,CAAE,EAAE,GAAK,EAAE,IAAM,SAAU,EAAG,CAAE,MAAO,IAAI,SAAQ,SAAU,EAAS,EAAQ,CAAE,EAAI,EAAE,GAAG,GAAI,EAAO,EAAS,EAAQ,EAAE,KAAM,EAAE,UAChJ,WAAgB,EAAS,EAAQ,EAAG,EAAG,CAAE,QAAQ,QAAQ,GAAG,KAAK,SAAS,EAAG,CAAE,EAAQ,CAAE,MAAO,EAAG,KAAM,KAAS,KAGtH,GAAuB,SAAU,EAAQ,EAAK,CAC1C,MAAI,QAAO,eAAkB,OAAO,eAAe,EAAQ,MAAO,CAAE,MAAO,IAAiB,EAAO,IAAM,EAClG,GAGX,GAAI,GAAqB,OAAO,OAAU,SAAS,EAAG,EAAG,CACrD,OAAO,eAAe,EAAG,UAAW,CAAE,WAAY,GAAM,MAAO,KAC9D,SAAS,EAAG,EAAG,CAChB,EAAE,QAAa,GAGnB,GAAe,SAAU,EAAK,CAC1B,GAAI,GAAO,EAAI,WAAY,MAAO,GAClC,GAAI,GAAS,GACb,GAAI,GAAO,KAAM,OAAS,KAAK,GAAK,AAAI,IAAM,WAAa,OAAO,UAAU,eAAe,KAAK,EAAK,IAAI,GAAgB,EAAQ,EAAK,GACtI,SAAmB,EAAQ,GACpB,GAGX,GAAkB,SAAU,EAAK,CAC7B,MAAQ,IAAO,EAAI,WAAc,EAAM,CAAE,QAAW,IAGxD,GAAyB,SAAU,EAAU,EAAY,CACrD,GAAI,CAAC,EAAW,IAAI,GAChB,KAAM,IAAI,WAAU,kDAExB,MAAO,GAAW,IAAI,IAG1B,GAAyB,SAAU,EAAU,EAAY,EAAO,CAC5D,GAAI,CAAC,EAAW,IAAI,GAChB,KAAM,IAAI,WAAU,kDAExB,SAAW,IAAI,EAAU,GAClB,GAGX,EAAS,YAAa,IACtB,EAAS,WAAY,IACrB,EAAS,SAAU,IACnB,EAAS,aAAc,IACvB,EAAS,UAAW,IACpB,EAAS,aAAc,IACvB,EAAS,YAAa,IACtB,EAAS,cAAe,IACxB,EAAS,eAAgB,IACzB,EAAS,kBAAmB,IAC5B,EAAS,WAAY,IACrB,EAAS,SAAU,IACnB,EAAS,WAAY,IACrB,EAAS,iBAAkB,IAC3B,EAAS,gBAAiB,IAC1B,EAAS,UAAW,IACpB,EAAS,mBAAoB,IAC7B,EAAS,mBAAoB,IAC7B,EAAS,gBAAiB,IAC1B,EAAS,uBAAwB,IACjC,EAAS,eAAgB,IACzB,EAAS,kBAAmB,IAC5B,EAAS,yBAA0B,IACnC,EAAS,yBAA0B,QC9SvC;AAAA;AAAA;AAAA;AAAA;AAAA,GAMA,AAAC,UAA0C,EAAM,EAAS,CACzD,AAAG,MAAO,KAAY,UAAY,MAAO,KAAW,SACnD,GAAO,QAAU,IACb,AAAG,MAAO,SAAW,YAAc,OAAO,IAC9C,OAAO,GAAI,GACP,AAAG,MAAO,KAAY,SAC1B,GAAQ,YAAiB,IAEzB,EAAK,YAAiB,MACrB,GAAM,UAAW,CACpB,MAAiB,WAAW,CAClB,GAAI,GAAuB,CAE/B,IACC,SAAS,EAAyB,EAAqB,EAAqB,CAEnF,aAGA,EAAoB,EAAE,EAAqB,CACzC,QAAW,UAAW,CAAE,MAAqB,OAI/C,GAAI,GAAe,EAAoB,KACnC,EAAoC,EAAoB,EAAE,GAE1D,EAAS,EAAoB,KAC7B,EAA8B,EAAoB,EAAE,GAEpD,EAAa,EAAoB,KACjC,EAA8B,EAAoB,EAAE,GAExD,WAAiB,EAAK,CAA6B,MAAI,OAAO,SAAW,YAAc,MAAO,QAAO,UAAa,SAAY,EAAU,SAAiB,EAAK,CAAE,MAAO,OAAO,IAAiB,EAAU,SAAiB,EAAK,CAAE,MAAO,IAAO,MAAO,SAAW,YAAc,EAAI,cAAgB,QAAU,IAAQ,OAAO,UAAY,SAAW,MAAO,IAAiB,EAAQ,GAEnX,WAAyB,EAAU,EAAa,CAAE,GAAI,CAAE,aAAoB,IAAgB,KAAM,IAAI,WAAU,qCAEhH,WAA2B,EAAQ,EAAO,CAAE,OAAS,GAAI,EAAG,EAAI,EAAM,OAAQ,IAAK,CAAE,GAAI,GAAa,EAAM,GAAI,EAAW,WAAa,EAAW,YAAc,GAAO,EAAW,aAAe,GAAU,SAAW,IAAY,GAAW,SAAW,IAAM,OAAO,eAAe,EAAQ,EAAW,IAAK,IAE7S,WAAsB,EAAa,EAAY,EAAa,CAAE,MAAI,IAAY,EAAkB,EAAY,UAAW,GAAiB,GAAa,EAAkB,EAAa,GAAqB,EAQzM,GAAI,GAA+B,UAAY,CAI7C,WAAyB,EAAS,CAChC,EAAgB,KAAM,GAEtB,KAAK,eAAe,GACpB,KAAK,gBAQP,SAAa,EAAiB,CAAC,CAC7B,IAAK,iBACL,MAAO,UAA0B,CAC/B,GAAI,GAAU,UAAU,OAAS,GAAK,UAAU,KAAO,OAAY,UAAU,GAAK,GAClF,KAAK,OAAS,EAAQ,OACtB,KAAK,UAAY,EAAQ,UACzB,KAAK,QAAU,EAAQ,QACvB,KAAK,OAAS,EAAQ,OACtB,KAAK,KAAO,EAAQ,KACpB,KAAK,QAAU,EAAQ,QACvB,KAAK,aAAe,KAOrB,CACD,IAAK,gBACL,MAAO,UAAyB,CAC9B,AAAI,KAAK,KACP,KAAK,aACI,KAAK,QACd,KAAK,iBAOR,CACD,IAAK,oBACL,MAAO,UAA6B,CAClC,GAAI,GAAQ,SAAS,gBAAgB,aAAa,SAAW,MAC7D,KAAK,SAAW,SAAS,cAAc,YAEvC,KAAK,SAAS,MAAM,SAAW,OAE/B,KAAK,SAAS,MAAM,OAAS,IAC7B,KAAK,SAAS,MAAM,QAAU,IAC9B,KAAK,SAAS,MAAM,OAAS,IAE7B,KAAK,SAAS,MAAM,SAAW,WAC/B,KAAK,SAAS,MAAM,EAAQ,QAAU,QAAU,UAEhD,GAAI,GAAY,OAAO,aAAe,SAAS,gBAAgB,UAC/D,YAAK,SAAS,MAAM,IAAM,GAAG,OAAO,EAAW,MAC/C,KAAK,SAAS,aAAa,WAAY,IACvC,KAAK,SAAS,MAAQ,KAAK,KACpB,KAAK,WAOb,CACD,IAAK,aACL,MAAO,UAAsB,CAC3B,GAAI,GAAQ,KAER,EAAW,KAAK,oBAEpB,KAAK,oBAAsB,UAAY,CACrC,MAAO,GAAM,cAGf,KAAK,YAAc,KAAK,UAAU,iBAAiB,QAAS,KAAK,sBAAwB,GACzF,KAAK,UAAU,YAAY,GAC3B,KAAK,aAAe,IAAiB,GACrC,KAAK,WACL,KAAK,eAON,CACD,IAAK,aACL,MAAO,UAAsB,CAC3B,AAAI,KAAK,aACP,MAAK,UAAU,oBAAoB,QAAS,KAAK,qBACjD,KAAK,YAAc,KACnB,KAAK,oBAAsB,MAGzB,KAAK,UACP,MAAK,UAAU,YAAY,KAAK,UAChC,KAAK,SAAW,QAOnB,CACD,IAAK,eACL,MAAO,UAAwB,CAC7B,KAAK,aAAe,IAAiB,KAAK,QAC1C,KAAK,aAMN,CACD,IAAK,WACL,MAAO,UAAoB,CACzB,GAAI,GAEJ,GAAI,CACF,EAAY,SAAS,YAAY,KAAK,cAC/B,EAAP,CACA,EAAY,GAGd,KAAK,aAAa,KAOnB,CACD,IAAK,eACL,MAAO,SAAsB,EAAW,CACtC,KAAK,QAAQ,KAAK,EAAY,UAAY,QAAS,CACjD,OAAQ,KAAK,OACb,KAAM,KAAK,aACX,QAAS,KAAK,QACd,eAAgB,KAAK,eAAe,KAAK,UAO5C,CACD,IAAK,iBACL,MAAO,UAA0B,CAC/B,AAAI,KAAK,SACP,KAAK,QAAQ,QAGf,SAAS,cAAc,OACvB,OAAO,eAAe,oBAOvB,CACD,IAAK,UAKL,MAAO,UAAmB,CACxB,KAAK,eAEN,CACD,IAAK,SACL,IAAK,UAAe,CAClB,GAAI,GAAS,UAAU,OAAS,GAAK,UAAU,KAAO,OAAY,UAAU,GAAK,OAGjF,GAFA,KAAK,QAAU,EAEX,KAAK,UAAY,QAAU,KAAK,UAAY,MAC9C,KAAM,IAAI,OAAM,uDAQpB,IAAK,UAAe,CAClB,MAAO,MAAK,UAQb,CACD,IAAK,SACL,IAAK,SAAa,EAAQ,CACxB,GAAI,IAAW,OACb,GAAI,GAAU,EAAQ,KAAY,UAAY,EAAO,WAAa,EAAG,CACnE,GAAI,KAAK,SAAW,QAAU,EAAO,aAAa,YAChD,KAAM,IAAI,OAAM,qFAGlB,GAAI,KAAK,SAAW,OAAU,GAAO,aAAa,aAAe,EAAO,aAAa,aACnF,KAAM,IAAI,OAAM,yGAGlB,KAAK,QAAU,MAEf,MAAM,IAAI,OAAM,gDAStB,IAAK,UAAe,CAClB,MAAO,MAAK,YAIT,KAGwB,EAAoB,EAErD,WAA0B,EAAK,CAA6B,MAAI,OAAO,SAAW,YAAc,MAAO,QAAO,UAAa,SAAY,EAAmB,SAAiB,EAAK,CAAE,MAAO,OAAO,IAAiB,EAAmB,SAAiB,EAAK,CAAE,MAAO,IAAO,MAAO,SAAW,YAAc,EAAI,cAAgB,QAAU,IAAQ,OAAO,UAAY,SAAW,MAAO,IAAiB,EAAiB,GAEvZ,WAAkC,EAAU,EAAa,CAAE,GAAI,CAAE,aAAoB,IAAgB,KAAM,IAAI,WAAU,qCAEzH,YAAoC,EAAQ,EAAO,CAAE,OAAS,GAAI,EAAG,EAAI,EAAM,OAAQ,IAAK,CAAE,GAAI,GAAa,EAAM,GAAI,EAAW,WAAa,EAAW,YAAc,GAAO,EAAW,aAAe,GAAU,SAAW,IAAY,GAAW,SAAW,IAAM,OAAO,eAAe,EAAQ,EAAW,IAAK,IAEtT,YAA+B,EAAa,EAAY,EAAa,CAAE,MAAI,IAAY,GAA2B,EAAY,UAAW,GAAiB,GAAa,GAA2B,EAAa,GAAqB,EAEpO,YAAmB,EAAU,EAAY,CAAE,GAAI,MAAO,IAAe,YAAc,IAAe,KAAQ,KAAM,IAAI,WAAU,sDAAyD,EAAS,UAAY,OAAO,OAAO,GAAc,EAAW,UAAW,CAAE,YAAa,CAAE,MAAO,EAAU,SAAU,GAAM,aAAc,MAAe,GAAY,GAAgB,EAAU,GAEnX,YAAyB,EAAG,EAAG,CAAE,UAAkB,OAAO,gBAAkB,SAAyB,EAAG,EAAG,CAAE,SAAE,UAAY,EAAU,GAAa,GAAgB,EAAG,GAErK,YAAsB,EAAS,CAAE,GAAI,GAA4B,KAA6B,MAAO,WAAgC,CAAE,GAAI,GAAQ,GAAgB,GAAU,EAAQ,GAAI,EAA2B,CAAE,GAAI,GAAY,GAAgB,MAAM,YAAa,EAAS,QAAQ,UAAU,EAAO,UAAW,OAAqB,GAAS,EAAM,MAAM,KAAM,WAAc,MAAO,IAA2B,KAAM,IAE5Z,YAAoC,EAAM,EAAM,CAAE,MAAI,IAAS,GAAiB,KAAU,UAAY,MAAO,IAAS,YAAsB,EAAe,GAAuB,GAElL,YAAgC,EAAM,CAAE,GAAI,IAAS,OAAU,KAAM,IAAI,gBAAe,6DAAgE,MAAO,GAE/J,aAAqC,CAA0E,GAApE,MAAO,UAAY,aAAe,CAAC,QAAQ,WAA6B,QAAQ,UAAU,KAAM,MAAO,GAAO,GAAI,MAAO,QAAU,WAAY,MAAO,GAAM,GAAI,CAAE,YAAK,UAAU,SAAS,KAAK,QAAQ,UAAU,KAAM,GAAI,UAAY,KAAa,SAAe,EAAP,CAAY,MAAO,IAE1T,YAAyB,EAAG,CAAE,UAAkB,OAAO,eAAiB,OAAO,eAAiB,SAAyB,EAAG,CAAE,MAAO,GAAE,WAAa,OAAO,eAAe,IAAc,GAAgB,GAWxM,YAA2B,EAAQ,EAAS,CAC1C,GAAI,GAAY,kBAAkB,OAAO,GAEzC,GAAI,EAAC,EAAQ,aAAa,GAI1B,MAAO,GAAQ,aAAa,GAQ9B,GAAI,IAAyB,SAAU,EAAU,CAC/C,GAAU,EAAW,GAErB,GAAI,GAAS,GAAa,GAM1B,WAAmB,EAAS,EAAS,CACnC,GAAI,GAEJ,SAAyB,KAAM,GAE/B,EAAQ,EAAO,KAAK,MAEpB,EAAM,eAAe,GAErB,EAAM,YAAY,GAEX,EAST,UAAsB,EAAW,CAAC,CAChC,IAAK,iBACL,MAAO,UAA0B,CAC/B,GAAI,GAAU,UAAU,OAAS,GAAK,UAAU,KAAO,OAAY,UAAU,GAAK,GAClF,KAAK,OAAS,MAAO,GAAQ,QAAW,WAAa,EAAQ,OAAS,KAAK,cAC3E,KAAK,OAAS,MAAO,GAAQ,QAAW,WAAa,EAAQ,OAAS,KAAK,cAC3E,KAAK,KAAO,MAAO,GAAQ,MAAS,WAAa,EAAQ,KAAO,KAAK,YACrE,KAAK,UAAY,EAAiB,EAAQ,aAAe,SAAW,EAAQ,UAAY,SAAS,OAOlG,CACD,IAAK,cACL,MAAO,SAAqB,EAAS,CACnC,GAAI,GAAS,KAEb,KAAK,SAAW,IAAiB,EAAS,QAAS,SAAU,GAAG,CAC9D,MAAO,GAAO,QAAQ,QAQzB,CACD,IAAK,UACL,MAAO,SAAiB,EAAG,CACzB,GAAI,GAAU,EAAE,gBAAkB,EAAE,cAEpC,AAAI,KAAK,iBACP,MAAK,gBAAkB,MAGzB,KAAK,gBAAkB,GAAI,GAAiB,CAC1C,OAAQ,KAAK,OAAO,GACpB,OAAQ,KAAK,OAAO,GACpB,KAAM,KAAK,KAAK,GAChB,UAAW,KAAK,UAChB,QAAS,EACT,QAAS,SAQZ,CACD,IAAK,gBACL,MAAO,SAAuB,EAAS,CACrC,MAAO,IAAkB,SAAU,KAOpC,CACD,IAAK,gBACL,MAAO,SAAuB,EAAS,CACrC,GAAI,GAAW,GAAkB,SAAU,GAE3C,GAAI,EACF,MAAO,UAAS,cAAc,KASjC,CACD,IAAK,cAML,MAAO,SAAqB,EAAS,CACnC,MAAO,IAAkB,OAAQ,KAMlC,CACD,IAAK,UACL,MAAO,UAAmB,CACxB,KAAK,SAAS,UAEV,KAAK,iBACP,MAAK,gBAAgB,UACrB,KAAK,gBAAkB,SAGzB,CAAC,CACH,IAAK,cACL,MAAO,UAAuB,CAC5B,GAAI,GAAS,UAAU,OAAS,GAAK,UAAU,KAAO,OAAY,UAAU,GAAK,CAAC,OAAQ,OACtF,EAAU,MAAO,IAAW,SAAW,CAAC,GAAU,EAClD,GAAU,CAAC,CAAC,SAAS,sBACzB,SAAQ,QAAQ,SAAU,GAAQ,CAChC,GAAU,IAAW,CAAC,CAAC,SAAS,sBAAsB,MAEjD,OAIJ,GACN,KAE8B,GAAa,IAIxC,IACC,SAAS,EAAQ,CAExB,GAAI,GAAqB,EAKzB,GAAI,MAAO,UAAY,aAAe,CAAC,QAAQ,UAAU,QAAS,CAC9D,GAAI,GAAQ,QAAQ,UAEpB,EAAM,QAAU,EAAM,iBACN,EAAM,oBACN,EAAM,mBACN,EAAM,kBACN,EAAM,sBAU1B,WAAkB,EAAS,EAAU,CACjC,KAAO,GAAW,EAAQ,WAAa,GAAoB,CACvD,GAAI,MAAO,GAAQ,SAAY,YAC3B,EAAQ,QAAQ,GAClB,MAAO,GAET,EAAU,EAAQ,YAI1B,EAAO,QAAU,GAKX,IACC,SAAS,EAAQ,EAA0B,EAAqB,CAEvE,GAAI,GAAU,EAAoB,KAYlC,WAAmB,EAAS,EAAU,EAAM,EAAU,EAAY,CAC9D,GAAI,GAAa,EAAS,MAAM,KAAM,WAEtC,SAAQ,iBAAiB,EAAM,EAAY,GAEpC,CACH,QAAS,UAAW,CAChB,EAAQ,oBAAoB,EAAM,EAAY,KAe1D,WAAkB,EAAU,EAAU,EAAM,EAAU,EAAY,CAE9D,MAAI,OAAO,GAAS,kBAAqB,WAC9B,EAAU,MAAM,KAAM,WAI7B,MAAO,IAAS,WAGT,EAAU,KAAK,KAAM,UAAU,MAAM,KAAM,WAIlD,OAAO,IAAa,UACpB,GAAW,SAAS,iBAAiB,IAIlC,MAAM,UAAU,IAAI,KAAK,EAAU,SAAU,EAAS,CACzD,MAAO,GAAU,EAAS,EAAU,EAAM,EAAU,MAa5D,WAAkB,EAAS,EAAU,EAAM,EAAU,CACjD,MAAO,UAAS,EAAG,CACf,EAAE,eAAiB,EAAQ,EAAE,OAAQ,GAEjC,EAAE,gBACF,EAAS,KAAK,EAAS,IAKnC,EAAO,QAAU,GAKX,IACC,SAAS,EAAyB,EAAS,CAQlD,EAAQ,KAAO,SAAS,EAAO,CAC3B,MAAO,KAAU,QACV,YAAiB,cACjB,EAAM,WAAa,GAS9B,EAAQ,SAAW,SAAS,EAAO,CAC/B,GAAI,GAAO,OAAO,UAAU,SAAS,KAAK,GAE1C,MAAO,KAAU,QACT,KAAS,qBAAuB,IAAS,4BACzC,UAAY,IACZ,GAAM,SAAW,GAAK,EAAQ,KAAK,EAAM,MASrD,EAAQ,OAAS,SAAS,EAAO,CAC7B,MAAO,OAAO,IAAU,UACjB,YAAiB,SAS5B,EAAQ,GAAK,SAAS,EAAO,CACzB,GAAI,GAAO,OAAO,UAAU,SAAS,KAAK,GAE1C,MAAO,KAAS,sBAMd,IACC,SAAS,EAAQ,EAA0B,EAAqB,CAEvE,GAAI,GAAK,EAAoB,KACzB,EAAW,EAAoB,KAWnC,WAAgB,EAAQ,EAAM,EAAU,CACpC,GAAI,CAAC,GAAU,CAAC,GAAQ,CAAC,EACrB,KAAM,IAAI,OAAM,8BAGpB,GAAI,CAAC,EAAG,OAAO,GACX,KAAM,IAAI,WAAU,oCAGxB,GAAI,CAAC,EAAG,GAAG,GACP,KAAM,IAAI,WAAU,qCAGxB,GAAI,EAAG,KAAK,GACR,MAAO,GAAW,EAAQ,EAAM,GAE/B,GAAI,EAAG,SAAS,GACjB,MAAO,GAAe,EAAQ,EAAM,GAEnC,GAAI,EAAG,OAAO,GACf,MAAO,GAAe,EAAQ,EAAM,GAGpC,KAAM,IAAI,WAAU,6EAa5B,WAAoB,EAAM,EAAM,EAAU,CACtC,SAAK,iBAAiB,EAAM,GAErB,CACH,QAAS,UAAW,CAChB,EAAK,oBAAoB,EAAM,KAc3C,WAAwB,EAAU,EAAM,EAAU,CAC9C,aAAM,UAAU,QAAQ,KAAK,EAAU,SAAS,EAAM,CAClD,EAAK,iBAAiB,EAAM,KAGzB,CACH,QAAS,UAAW,CAChB,MAAM,UAAU,QAAQ,KAAK,EAAU,SAAS,EAAM,CAClD,EAAK,oBAAoB,EAAM,OAe/C,WAAwB,EAAU,EAAM,EAAU,CAC9C,MAAO,GAAS,SAAS,KAAM,EAAU,EAAM,GAGnD,EAAO,QAAU,GAKX,IACC,SAAS,EAAQ,CAExB,WAAgB,EAAS,CACrB,GAAI,GAEJ,GAAI,EAAQ,WAAa,SACrB,EAAQ,QAER,EAAe,EAAQ,cAElB,EAAQ,WAAa,SAAW,EAAQ,WAAa,WAAY,CACtE,GAAI,GAAa,EAAQ,aAAa,YAEtC,AAAK,GACD,EAAQ,aAAa,WAAY,IAGrC,EAAQ,SACR,EAAQ,kBAAkB,EAAG,EAAQ,MAAM,QAEtC,GACD,EAAQ,gBAAgB,YAG5B,EAAe,EAAQ,UAEtB,CACD,AAAI,EAAQ,aAAa,oBACrB,EAAQ,QAGZ,GAAI,GAAY,OAAO,eACnB,EAAQ,SAAS,cAErB,EAAM,mBAAmB,GACzB,EAAU,kBACV,EAAU,SAAS,GAEnB,EAAe,EAAU,WAG7B,MAAO,GAGX,EAAO,QAAU,GAKX,IACC,SAAS,EAAQ,CAExB,YAAc,EAKd,EAAE,UAAY,CACZ,GAAI,SAAU,EAAM,EAAU,EAAK,CACjC,GAAI,GAAI,KAAK,GAAM,MAAK,EAAI,IAE5B,MAAC,GAAE,IAAU,GAAE,GAAQ,KAAK,KAAK,CAC/B,GAAI,EACJ,IAAK,IAGA,MAGT,KAAM,SAAU,EAAM,EAAU,EAAK,CACnC,GAAI,GAAO,KACX,YAAqB,CACnB,EAAK,IAAI,EAAM,GACf,EAAS,MAAM,EAAK,WAGtB,SAAS,EAAI,EACN,KAAK,GAAG,EAAM,EAAU,IAGjC,KAAM,SAAU,EAAM,CACpB,GAAI,GAAO,GAAG,MAAM,KAAK,UAAW,GAChC,EAAW,OAAK,GAAM,MAAK,EAAI,KAAK,IAAS,IAAI,QACjD,EAAI,EACJ,EAAM,EAAO,OAEjB,IAAK,EAAG,EAAI,EAAK,IACf,EAAO,GAAG,GAAG,MAAM,EAAO,GAAG,IAAK,GAGpC,MAAO,OAGT,IAAK,SAAU,EAAM,EAAU,CAC7B,GAAI,GAAI,KAAK,GAAM,MAAK,EAAI,IACxB,EAAO,EAAE,GACT,EAAa,GAEjB,GAAI,GAAQ,EACV,OAAS,GAAI,EAAG,EAAM,EAAK,OAAQ,EAAI,EAAK,IAC1C,AAAI,EAAK,GAAG,KAAO,GAAY,EAAK,GAAG,GAAG,IAAM,GAC9C,EAAW,KAAK,EAAK,IAQ3B,MAAC,GAAW,OACR,EAAE,GAAQ,EACV,MAAO,GAAE,GAEN,OAIX,EAAO,QAAU,EACjB,EAAO,QAAQ,YAAc,IAQf,EAA2B,GAG/B,WAA6B,EAAU,CAEtC,GAAG,EAAyB,GAC3B,MAAO,GAAyB,GAAU,QAG3C,GAAI,GAAS,EAAyB,GAAY,CAGjD,QAAS,IAIV,SAAoB,GAAU,EAAQ,EAAO,QAAS,GAG/C,EAAO,QAKf,MAAC,WAAW,CAEX,EAAoB,EAAI,SAAS,EAAQ,CACxC,GAAI,GAAS,GAAU,EAAO,WAC7B,UAAW,CAAE,MAAO,GAAO,SAC3B,UAAW,CAAE,MAAO,IACrB,SAAoB,EAAE,EAAQ,CAAE,EAAG,IAC5B,MAKR,UAAW,CAEX,EAAoB,EAAI,SAAS,EAAS,EAAY,CACrD,OAAQ,KAAO,GACd,AAAG,EAAoB,EAAE,EAAY,IAAQ,CAAC,EAAoB,EAAE,EAAS,IAC5E,OAAO,eAAe,EAAS,EAAK,CAAE,WAAY,GAAM,IAAK,EAAW,SAO3E,UAAW,CACX,EAAoB,EAAI,SAAS,EAAK,EAAM,CAAE,MAAO,QAAO,UAAU,eAAe,KAAK,EAAK,OAOzF,EAAoB,QAEpC,YCx7BD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAQA,aAOA,GAAI,IAAkB,UAOtB,GAAO,QAAU,GAUjB,YAAoB,EAAQ,CAC1B,GAAI,GAAM,GAAK,EACX,EAAQ,GAAgB,KAAK,GAEjC,GAAI,CAAC,EACH,MAAO,GAGT,GAAI,GACA,EAAO,GACP,EAAQ,EACR,EAAY,EAEhB,IAAK,EAAQ,EAAM,MAAO,EAAQ,EAAI,OAAQ,IAAS,CACrD,OAAQ,EAAI,WAAW,QAChB,IACH,EAAS,SACT,UACG,IACH,EAAS,QACT,UACG,IACH,EAAS,QACT,UACG,IACH,EAAS,OACT,UACG,IACH,EAAS,OACT,cAEA,SAGJ,AAAI,IAAc,GAChB,IAAQ,EAAI,UAAU,EAAW,IAGnC,EAAY,EAAQ,EACpB,GAAQ,EAGV,MAAO,KAAc,EACjB,EAAO,EAAI,UAAU,EAAW,GAChC,KCtDN,OAAO,SCtBP,OAAkB,SACZ,CACF,YACA,YACA,UACA,cACA,WACA,cACA,aACA,eACA,gBACA,mBACA,YACA,SACA,YACA,kBACA,gBACA,WACA,oBACA,oBACA,iBACA,wBACA,gBACA,mBACA,0BACA,2BACA,WCtBE,WAAqB,EAAU,CACnC,MAAO,OAAO,IAAU,WCIpB,YAA8B,EAAgC,CAClE,GAAM,GAAS,SAAC,EAAa,CAC3B,MAAM,KAAK,GACX,EAAS,MAAQ,GAAI,SAAQ,OAGzB,EAAW,EAAW,GAC5B,SAAS,UAAY,OAAO,OAAO,MAAM,WACzC,EAAS,UAAU,YAAc,EAC1B,ECAF,GAAM,IAA+C,GAC1D,SAAC,EAAM,CACL,MAAA,UAA4C,EAA0B,CACpE,EAAO,MACP,KAAK,QAAU,EACR,EAAO,OAAM;EACxB,EAAO,IAAI,SAAC,EAAK,EAAC,CAAK,MAAG,GAAI,EAAC,KAAK,EAAI,aAAc,KAAK;KACnD,GACJ,KAAK,KAAO,sBACZ,KAAK,OAAS,KCtBd,YAAuB,EAA6B,EAAO,CAC/D,GAAI,EAAK,CACP,GAAM,GAAQ,EAAI,QAAQ,GAC1B,GAAK,GAAS,EAAI,OAAO,EAAO,ICSpC,GAAA,IAAA,UAAA,CAyBE,WAAoB,EAA4B,CAA5B,KAAA,gBAAA,EAdb,KAAA,OAAS,GAER,KAAA,WAAmD,KAMnD,KAAA,WAAoD,KAc5D,SAAA,UAAA,YAAA,UAAA,aACM,EAEJ,GAAI,CAAC,KAAK,OAAQ,CAChB,KAAK,OAAS,GAGN,GAAA,GAAe,KAAI,WAC3B,GAAI,EAEF,GADA,KAAK,WAAa,KACd,MAAM,QAAQ,OAChB,OAAqB,GAAA,GAAA,GAAU,EAAA,EAAA,OAAA,CAAA,EAAA,KAAA,EAAA,EAAA,OAAE,CAA5B,GAAM,GAAM,EAAA,MACf,EAAO,OAAO,4GAGhB,GAAW,OAAO,MAId,GAAA,GAAoB,KAAI,gBAChC,GAAI,EAAW,GACb,GAAI,CACF,UACO,EAAP,CACA,EAAS,YAAa,IAAsB,EAAE,OAAS,CAAC,GAIpD,GAAA,GAAe,KAAI,WAC3B,GAAI,EAAY,CACd,KAAK,WAAa,SAClB,OAAuB,GAAA,GAAA,GAAU,EAAA,EAAA,OAAA,CAAA,EAAA,KAAA,EAAA,EAAA,OAAE,CAA9B,GAAM,GAAQ,EAAA,MACjB,GAAI,CACF,GAAa,SACN,EAAP,CACA,EAAS,GAAM,KAAN,EAAU,GACnB,AAAI,YAAe,IACjB,EAAM,EAAA,EAAA,GAAA,EAAO,IAAM,EAAK,EAAI,SAE5B,EAAO,KAAK,uGAMpB,GAAI,EACF,KAAM,IAAI,IAAoB,KAuBpC,EAAA,UAAA,IAAA,SAAI,EAAuB,OAGzB,GAAI,GAAY,IAAa,KAC3B,GAAI,KAAK,OAGP,GAAa,OACR,CACL,GAAI,YAAoB,GAAc,CAGpC,GAAI,EAAS,QAAU,EAAS,WAAW,MACzC,OAEF,EAAS,WAAW,MAEtB,AAAC,MAAK,WAAa,GAAA,KAAK,cAAU,MAAA,IAAA,OAAA,EAAI,IAAI,KAAK,KAU7C,EAAA,UAAA,WAAR,SAAmB,EAAoB,CAC7B,GAAA,GAAe,KAAI,WAC3B,MAAO,KAAe,GAAW,MAAM,QAAQ,IAAe,EAAW,SAAS,IAU5E,EAAA,UAAA,WAAR,SAAmB,EAAoB,CAC7B,GAAA,GAAe,KAAI,WAC3B,KAAK,WAAa,MAAM,QAAQ,GAAe,GAAW,KAAK,GAAS,GAAc,EAAa,CAAC,EAAY,GAAU,GAOpH,EAAA,UAAA,cAAR,SAAsB,EAAoB,CAChC,GAAA,GAAe,KAAI,WAC3B,AAAI,IAAe,EACjB,KAAK,WAAa,KACT,MAAM,QAAQ,IACvB,GAAU,EAAY,IAkB1B,EAAA,UAAA,OAAA,SAAO,EAAsC,CACnC,GAAA,GAAe,KAAI,WAC3B,GAAc,GAAU,EAAY,GAEhC,YAAoB,IACtB,EAAS,cAAc,OAhLb,EAAA,MAAS,UAAA,CACrB,GAAM,GAAQ,GAAI,GAClB,SAAM,OAAS,GACR,KAgLX,KAEO,GAAM,IAAqB,GAAa,MAEzC,YAAyB,EAAU,CACvC,MACE,aAAiB,KAChB,GAAS,UAAY,IAAS,EAAW,EAAM,SAAW,EAAW,EAAM,MAAQ,EAAW,EAAM,aAIzG,YAAsB,EAAuC,CAC3D,AAAI,EAAW,GACb,IAEA,EAAS,cC9MN,GAAM,IAAuB,CAClC,iBAAkB,KAClB,sBAAuB,KACvB,QAAS,OACT,sCAAuC,GACvC,yBAA0B,ICErB,GAAM,IAAmC,CAG9C,WAAU,UAAA,QAAC,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GACD,GAAA,GAAa,GAAe,SACpC,MAAQ,KAAQ,KAAA,OAAR,EAAU,aAAc,YAAW,MAAA,OAAA,EAAA,GAAA,EAAI,MAEjD,aAAY,SAAC,EAAM,CACT,GAAA,GAAa,GAAe,SACpC,MAAQ,KAAQ,KAAA,OAAR,EAAU,eAAgB,cAAc,IAElD,SAAU,QCbN,YAA+B,EAAQ,CAC3C,GAAgB,WAAW,UAAA,CACjB,GAAA,GAAqB,GAAM,iBACnC,GAAI,EAEF,EAAiB,OAGjB,MAAM,KCnBN,YAAc,ECMb,GAAM,IAAyB,UAAA,CAAM,MAAA,IAAmB,IAAK,OAAW,WAOzE,YAA4B,EAAU,CAC1C,MAAO,IAAmB,IAAK,OAAW,GAQtC,YAA8B,EAAQ,CAC1C,MAAO,IAAmB,IAAK,EAAO,QASlC,YAA6B,EAAuB,EAAY,EAAU,CAC9E,MAAO,CACL,KAAI,EACJ,MAAK,EACL,MAAK,GCnCT,GAAI,IAAuD,KASrD,YAAuB,EAAc,CACzC,GAAI,GAAO,sCAAuC,CAChD,GAAM,GAAS,CAAC,GAKhB,GAJI,GACF,IAAU,CAAE,YAAa,GAAO,MAAO,OAEzC,IACI,EAAQ,CACJ,GAAA,GAAyB,GAAvB,EAAW,EAAA,YAAE,EAAK,EAAA,MAE1B,GADA,GAAU,KACN,EACF,KAAM,QAMV,KAQE,YAAuB,EAAQ,CACnC,AAAI,GAAO,uCAAyC,IAClD,IAAQ,YAAc,GACtB,GAAQ,MAAQ,GCnBpB,GAAA,IAAA,SAAA,EAAA,CAAmC,EAAA,EAAA,GA6BjC,WAAY,EAA6C,CAAzD,GAAA,GACE,EAAA,KAAA,OAAO,KATC,SAAA,UAAqB,GAU7B,AAAI,EACF,GAAK,YAAc,EAGf,GAAe,IACjB,EAAY,IAAI,IAGlB,EAAK,YAAc,KAvBhB,SAAA,OAAP,SAAiB,EAAwB,EAA2B,EAAqB,CACvF,MAAO,IAAI,IAAe,EAAM,EAAO,IAiCzC,EAAA,UAAA,KAAA,SAAK,EAAS,CACZ,AAAI,KAAK,UACP,GAA0B,GAAiB,GAAQ,MAEnD,KAAK,MAAM,IAWf,EAAA,UAAA,MAAA,SAAM,EAAS,CACb,AAAI,KAAK,UACP,GAA0B,GAAkB,GAAM,MAElD,MAAK,UAAY,GACjB,KAAK,OAAO,KAUhB,EAAA,UAAA,SAAA,UAAA,CACE,AAAI,KAAK,UACP,GAA0B,GAAuB,MAEjD,MAAK,UAAY,GACjB,KAAK,cAIT,EAAA,UAAA,YAAA,UAAA,CACE,AAAK,KAAK,QACR,MAAK,UAAY,GACjB,EAAA,UAAM,YAAW,KAAA,MACjB,KAAK,YAAc,OAIb,EAAA,UAAA,MAAV,SAAgB,EAAQ,CACtB,KAAK,YAAY,KAAK,IAGd,EAAA,UAAA,OAAV,SAAiB,EAAQ,CACvB,GAAI,CACF,KAAK,YAAY,MAAM,WAEvB,KAAK,gBAIC,EAAA,UAAA,UAAV,UAAA,CACE,GAAI,CACF,KAAK,YAAY,mBAEjB,KAAK,gBAGX,GApHmC,IAsHnC,GAAA,IAAA,SAAA,EAAA,CAAuC,EAAA,EAAA,GACrC,WACE,EACA,EACA,EAA8B,CAHhC,GAAA,GAKE,EAAA,KAAA,OAAO,KAEH,EACJ,GAAI,EAAW,GAGb,EAAO,UACE,EAAgB,CAMzB,AAAG,EAA0B,EAAc,KAAlC,EAAoB,EAAc,MAA3B,EAAa,EAAc,SAC3C,GAAI,GACJ,AAAI,GAAQ,GAAO,yBAIjB,GAAU,OAAO,OAAO,GACxB,EAAQ,YAAc,UAAA,CAAM,MAAA,GAAK,gBAEjC,EAAU,EAEZ,EAAO,GAAI,KAAA,OAAJ,EAAM,KAAK,GAClB,EAAQ,GAAK,KAAA,OAAL,EAAO,KAAK,GACpB,EAAW,GAAQ,KAAA,OAAR,EAAU,KAAK,GAK5B,SAAK,YAAc,CACjB,KAAM,EAAO,GAAqB,EAAM,GAAQ,EAChD,MAAO,GAAqB,GAAK,KAAL,EAAS,GAAqB,GAC1D,SAAU,EAAW,GAAqB,EAAU,GAAQ,KAGlE,MAAA,IA3CuC,IAoDvC,YAA8B,EAA8B,EAA6B,CACvF,MAAO,WAAA,QAAC,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GACN,GAAI,CACF,EAAO,MAAA,OAAA,EAAA,GAAA,EAAI,WACJ,EAAP,CACA,AAAI,GAAO,sCACT,GAAa,GAIb,GAAqB,KAW7B,YAA6B,EAAQ,CACnC,KAAM,GAQR,YAAmC,EAA2C,EAA2B,CAC/F,GAAA,GAA0B,GAAM,sBACxC,GAAyB,GAAgB,WAAW,UAAA,CAAM,MAAA,GAAsB,EAAc,KAQzF,GAAM,IAA6D,CACxE,OAAQ,GACR,KAAM,EACN,MAAO,GACP,SAAU,GCzOL,GAAM,IAA+B,UAAA,CAAM,MAAC,OAAO,SAAW,YAAc,OAAO,YAAe,kBCDnG,YAAsB,EAAI,CAC9B,MAAO,GCsEH,aAAc,QAAC,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GACnB,MAAO,IAAc,GAIjB,YAA8B,EAA+B,CACjE,MAAI,GAAI,SAAW,EACV,GAGL,EAAI,SAAW,EACV,EAAI,GAGN,SAAe,EAAQ,CAC5B,MAAO,GAAI,OAAO,SAAC,EAAW,EAAuB,CAAK,MAAA,GAAG,IAAO,ICnExE,GAAA,GAAA,UAAA,CAkBE,WAAY,EAA6E,CACvF,AAAI,GACF,MAAK,WAAa,GA8BtB,SAAA,UAAA,KAAA,SAAQ,EAAyB,CAC/B,GAAM,GAAa,GAAI,GACvB,SAAW,OAAS,KACpB,EAAW,SAAW,EACf,GA2IT,EAAA,UAAA,UAAA,SACE,EACA,EACA,EAA8B,CAHhC,GAAA,GAAA,KAKQ,EAAa,GAAa,GAAkB,EAAiB,GAAI,IAAe,EAAgB,EAAO,GAE7G,UAAa,UAAA,CACL,GAAA,GAAuB,EAArB,EAAQ,EAAA,SAAE,EAAM,EAAA,OACxB,EAAW,IACT,EAGI,EAAS,KAAK,EAAY,GAC1B,EAIA,EAAK,WAAW,GAGhB,EAAK,cAAc,MAIpB,GAIC,EAAA,UAAA,cAAV,SAAwB,EAAmB,CACzC,GAAI,CACF,MAAO,MAAK,WAAW,SAChB,EAAP,CAIA,EAAK,MAAM,KA+Df,EAAA,UAAA,QAAA,SAAQ,EAA0B,EAAoC,CAAtE,GAAA,GAAA,KACE,SAAc,GAAe,GAEtB,GAAI,GAAkB,SAAC,EAAS,EAAM,CAG3C,GAAI,GACJ,EAAe,EAAK,UAClB,SAAC,EAAK,CACJ,GAAI,CACF,EAAK,SACE,EAAP,CACA,EAAO,GACP,GAAY,MAAZ,EAAc,gBAGlB,EACA,MAMI,EAAA,UAAA,WAAV,SAAqB,EAA2B,OAC9C,MAAO,GAAA,KAAK,UAAM,MAAA,IAAA,OAAA,OAAA,EAAE,UAAU,IAQhC,EAAA,UAAC,IAAD,UAAA,CACE,MAAO,OA6FT,EAAA,UAAA,KAAA,UAAA,QAAK,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GACH,MAAO,IAAc,GAAY,OA8BnC,EAAA,UAAA,UAAA,SAAU,EAAoC,CAA9C,GAAA,GAAA,KACE,SAAc,GAAe,GAEtB,GAAI,GAAY,SAAC,EAAS,EAAM,CACrC,GAAI,GACJ,EAAK,UACH,SAAC,EAAI,CAAK,MAAC,GAAQ,GACnB,SAAC,EAAQ,CAAK,MAAA,GAAO,IACrB,UAAA,CAAM,MAAA,GAAQ,QAtab,EAAA,OAAkC,SAAI,EAAwD,CACnG,MAAO,IAAI,GAAc,IAya7B,KASA,YAAwB,EAA+C,OACrE,MAAO,GAAA,GAAW,KAAX,EAAe,GAAO,WAAO,MAAA,IAAA,OAAA,EAAI,QAG1C,YAAuB,EAAU,CAC/B,MAAO,IAAS,EAAW,EAAM,OAAS,EAAW,EAAM,QAAU,EAAW,EAAM,UAGxF,YAAyB,EAAU,CACjC,MAAQ,IAAS,YAAiB,KAAgB,GAAW,IAAU,GAAe,GC1elF,YAAkB,EAAW,CACjC,MAAO,GAAW,GAAM,KAAA,OAAN,EAAQ,MAOtB,WACJ,EAAqF,CAErF,MAAO,UAAC,EAAqB,CAC3B,GAAI,GAAQ,GACV,MAAO,GAAO,KAAK,SAA+B,EAA2B,CAC3E,GAAI,CACF,MAAO,GAAK,EAAc,YACnB,EAAP,CACA,KAAK,MAAM,MAIjB,KAAM,IAAI,WAAU,2CCvBxB,GAAA,GAAA,SAAA,EAAA,CAA2C,EAAA,EAAA,GAazC,WACE,EACA,EACA,EACA,EACQ,EAAuB,CALjC,GAAA,GAmBE,EAAA,KAAA,KAAM,IAAY,KAdV,SAAA,WAAA,EAeR,EAAK,MAAQ,EACT,SAAuC,EAAQ,CAC7C,GAAI,CACF,EAAO,SACA,EAAP,CACA,EAAY,MAAM,KAGtB,EAAA,UAAM,MACV,EAAK,OAAS,EACV,SAAuC,EAAQ,CAC7C,GAAI,CACF,EAAQ,SACD,EAAP,CAEA,EAAY,MAAM,WAGlB,KAAK,gBAGT,EAAA,UAAM,OACV,EAAK,UAAY,EACb,UAAA,CACE,GAAI,CACF,UACO,EAAP,CAEA,EAAY,MAAM,WAGlB,KAAK,gBAGT,EAAA,UAAM,YAGZ,SAAA,UAAA,YAAA,UAAA,OACU,EAAW,KAAI,OACvB,EAAA,UAAM,YAAW,KAAA,MAEjB,CAAC,GAAU,IAAA,KAAK,cAAU,MAAA,IAAA,QAAA,EAAA,KAAf,QAEf,GA5E2C,ICQpC,GAAM,IAAiD,CAG5D,SAAA,SAAS,EAAQ,CACf,GAAI,GAAU,sBACV,EAAkD,qBAC9C,EAAa,GAAsB,SAC3C,AAAI,GACF,GAAU,EAAS,sBACnB,EAAS,EAAS,sBAEpB,GAAM,GAAS,EAAQ,SAAC,EAAS,CAI/B,EAAS,OACT,EAAS,KAEX,MAAO,IAAI,IAAa,UAAA,CAAM,MAAA,IAAM,KAAA,OAAN,EAAS,MAEzC,sBAAqB,UAAA,QAAC,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GACZ,GAAA,GAAa,GAAsB,SAC3C,MAAQ,KAAQ,KAAA,OAAR,EAAU,wBAAyB,uBAAsB,MAAA,OAAA,EAAA,GAAA,EAAI,MAEvE,qBAAoB,UAAA,QAAC,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GACX,GAAA,GAAa,GAAsB,SAC3C,MAAQ,KAAQ,KAAA,OAAR,EAAU,uBAAwB,sBAAqB,MAAA,OAAA,EAAA,GAAA,EAAI,MAErE,SAAU,QCrBL,GAAM,IAAuD,GAClE,SAAC,EAAM,CACL,MAAA,WAAoC,CAClC,EAAO,MACP,KAAK,KAAO,0BACZ,KAAK,QAAU,yBCVrB,GAAA,GAAA,SAAA,EAAA,CAAgC,EAAA,EAAA,GAqB9B,YAAA,CAAA,GAAA,GAEE,EAAA,KAAA,OAAO,KAtBT,SAAA,OAAS,GAET,EAAA,UAA2B,GAE3B,EAAA,UAAY,GAEZ,EAAA,SAAW,GAEX,EAAA,YAAmB,OAkBnB,SAAA,UAAA,KAAA,SAAQ,EAAwB,CAC9B,GAAM,GAAU,GAAI,IAAiB,KAAM,MAC3C,SAAQ,SAAW,EACZ,GAIC,EAAA,UAAA,eAAV,UAAA,CACE,GAAI,KAAK,OACP,KAAM,IAAI,KAId,EAAA,UAAA,KAAA,SAAK,EAAQ,CAAb,GAAA,GAAA,KACE,GAAa,UAAA,SAEX,GADA,EAAK,iBACD,CAAC,EAAK,UAAW,CACnB,GAAM,GAAO,EAAK,UAAU,YAC5B,OAAuB,GAAA,GAAA,GAAI,EAAA,EAAA,OAAA,CAAA,EAAA,KAAA,EAAA,EAAA,OAAE,CAAxB,GAAM,GAAQ,EAAA,MACjB,EAAS,KAAK,0GAMtB,EAAA,UAAA,MAAA,SAAM,EAAQ,CAAd,GAAA,GAAA,KACE,GAAa,UAAA,CAEX,GADA,EAAK,iBACD,CAAC,EAAK,UAAW,CACnB,EAAK,SAAW,EAAK,UAAY,GACjC,EAAK,YAAc,EAEnB,OADQ,GAAc,EAAI,UACnB,EAAU,QACf,EAAU,QAAS,MAAM,OAMjC,EAAA,UAAA,SAAA,UAAA,CAAA,GAAA,GAAA,KACE,GAAa,UAAA,CAEX,GADA,EAAK,iBACD,CAAC,EAAK,UAAW,CACnB,EAAK,UAAY,GAEjB,OADQ,GAAc,EAAI,UACnB,EAAU,QACf,EAAU,QAAS,eAM3B,EAAA,UAAA,YAAA,UAAA,CACE,KAAK,UAAY,KAAK,OAAS,GAC/B,KAAK,UAAY,MAGnB,OAAA,eAAI,EAAA,UAAA,WAAQ,KAAZ,UAAA,OACE,MAAO,IAAA,KAAK,aAAS,MAAA,IAAA,OAAA,OAAA,EAAE,QAAS,mCAIxB,EAAA,UAAA,cAAV,SAAwB,EAAyB,CAC/C,YAAK,iBACE,EAAA,UAAM,cAAa,KAAA,KAAC,IAInB,EAAA,UAAA,WAAV,SAAqB,EAAyB,CAC5C,YAAK,iBACL,KAAK,wBAAwB,GACtB,KAAK,gBAAgB,IAIpB,EAAA,UAAA,gBAAV,SAA0B,EAA2B,CAC7C,GAAA,GAAqC,KAAnC,EAAQ,EAAA,SAAE,EAAS,EAAA,UAAE,EAAS,EAAA,UACtC,MAAO,IAAY,EACf,GACC,GAAU,KAAK,GAAa,GAAI,IAAa,UAAA,CAAM,MAAA,IAAU,EAAW,OAIrE,EAAA,UAAA,wBAAV,SAAkC,EAA2B,CACrD,GAAA,GAAuC,KAArC,EAAQ,EAAA,SAAE,EAAW,EAAA,YAAE,EAAS,EAAA,UACxC,AAAI,EACF,EAAW,MAAM,GACR,GACT,EAAW,YAUf,EAAA,UAAA,aAAA,UAAA,CACE,GAAM,GAAkB,GAAI,GAC5B,SAAW,OAAS,KACb,GA/GF,EAAA,OAAkC,SAAI,EAA0B,EAAqB,CAC1F,MAAO,IAAI,IAAoB,EAAa,IAgHhD,GAlIgC,GAuIhC,GAAA,IAAA,SAAA,EAAA,CAAyC,EAAA,EAAA,GACvC,WAES,EACP,EAAsB,CAHxB,GAAA,GAKE,EAAA,KAAA,OAAO,KAHA,SAAA,YAAA,EAIP,EAAK,OAAS,IAGhB,SAAA,UAAA,KAAA,SAAK,EAAQ,SACX,AAAA,GAAA,GAAA,KAAK,eAAW,MAAA,IAAA,OAAA,OAAA,EAAE,QAAI,MAAA,IAAA,QAAA,EAAA,KAAA,EAAG,IAG3B,EAAA,UAAA,MAAA,SAAM,EAAQ,SACZ,AAAA,GAAA,GAAA,KAAK,eAAW,MAAA,IAAA,OAAA,OAAA,EAAE,SAAK,MAAA,IAAA,QAAA,EAAA,KAAA,EAAG,IAG5B,EAAA,UAAA,SAAA,UAAA,SACE,AAAA,GAAA,GAAA,KAAK,eAAW,MAAA,IAAA,OAAA,OAAA,EAAE,YAAQ,MAAA,IAAA,QAAA,EAAA,KAAA,IAIlB,EAAA,UAAA,WAAV,SAAqB,EAAyB,SAC5C,MAAO,GAAA,GAAA,KAAK,UAAM,MAAA,IAAA,OAAA,OAAA,EAAE,UAAU,MAAW,MAAA,IAAA,OAAA,EAAI,IAEjD,GA1ByC,GCjJlC,GAAM,IAA+C,CAC1D,IAAG,UAAA,CAGD,MAAQ,IAAsB,UAAY,MAAM,OAElD,SAAU,QCwBZ,GAAA,IAAA,SAAA,EAAA,CAAsC,EAAA,EAAA,GAUpC,WACU,EACA,EACA,EAA6D,CAF7D,AAAA,IAAA,QAAA,GAAA,KACA,IAAA,QAAA,GAAA,KACA,IAAA,QAAA,GAAA,IAHV,GAAA,GAKE,EAAA,KAAA,OAAO,KAJC,SAAA,YAAA,EACA,EAAA,YAAA,EACA,EAAA,mBAAA,EAZF,EAAA,QAA0B,GAC1B,EAAA,oBAAsB,GAc5B,EAAK,oBAAsB,IAAgB,IAC3C,EAAK,YAAc,KAAK,IAAI,EAAG,GAC/B,EAAK,YAAc,KAAK,IAAI,EAAG,KAGjC,SAAA,UAAA,KAAA,SAAK,EAAQ,CACL,GAAA,GAA+E,KAA7E,EAAS,EAAA,UAAE,EAAO,EAAA,QAAE,EAAmB,EAAA,oBAAE,EAAkB,EAAA,mBAAE,EAAW,EAAA,YAChF,AAAK,GACH,GAAQ,KAAK,GACb,CAAC,GAAuB,EAAQ,KAAK,EAAmB,MAAQ,IAElE,KAAK,cACL,EAAA,UAAM,KAAI,KAAA,KAAC,IAIH,EAAA,UAAA,WAAV,SAAqB,EAAyB,CAC5C,KAAK,iBACL,KAAK,cAQL,OANM,GAAe,KAAK,gBAAgB,GAEpC,EAAmC,KAAjC,EAAmB,EAAA,oBAAE,EAAO,EAAA,QAG9B,EAAO,EAAQ,QACZ,EAAI,EAAG,EAAI,EAAK,QAAU,CAAC,EAAW,OAAQ,GAAK,EAAsB,EAAI,EACpF,EAAW,KAAK,EAAK,IAGvB,YAAK,wBAAwB,GAEtB,GAGD,EAAA,UAAA,YAAR,UAAA,CACQ,GAAA,GAAoE,KAAlE,EAAW,EAAA,YAAE,EAAkB,EAAA,mBAAE,EAAO,EAAA,QAAE,EAAmB,EAAA,oBAK/D,EAAsB,GAAsB,EAAI,GAAK,EAK3D,GAJA,EAAc,KAAY,EAAqB,EAAQ,QAAU,EAAQ,OAAO,EAAG,EAAQ,OAAS,GAIhG,CAAC,EAAqB,CAKxB,OAJM,GAAM,EAAmB,MAC3B,EAAO,EAGF,EAAI,EAAG,EAAI,EAAQ,QAAW,EAAQ,IAAiB,EAAK,GAAK,EACxE,EAAO,EAET,GAAQ,EAAQ,OAAO,EAAG,EAAO,KAGvC,GAzEsC,GClBtC,GAAA,IAAA,SAAA,EAAA,CAA+B,EAAA,EAAA,GAC7B,WAAY,EAAsB,EAAmD,OACnF,GAAA,KAAA,OAAO,KAYF,SAAA,UAAA,SAAP,SAAgB,EAAW,EAAiB,CAAjB,MAAA,KAAA,QAAA,GAAA,GAClB,MAEX,GAjB+B,ICJxB,GAAM,IAAqC,CAGhD,YAAW,UAAA,QAAC,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GACF,GAAA,GAAa,GAAgB,SACrC,MAAQ,KAAQ,KAAA,OAAR,EAAU,cAAe,aAAY,MAAA,OAAA,EAAA,GAAA,EAAI,MAEnD,cAAa,SAAC,EAAM,CACV,GAAA,GAAa,GAAgB,SACrC,MAAQ,KAAQ,KAAA,OAAR,EAAU,gBAAiB,eAAe,IAEpD,SAAU,QClBZ,GAAA,IAAA,SAAA,EAAA,CAAoC,EAAA,EAAA,GAOlC,WAAsB,EAAqC,EAAmD,CAA9G,GAAA,GACE,EAAA,KAAA,KAAM,EAAW,IAAK,KADF,SAAA,UAAA,EAAqC,EAAA,KAAA,EAFjD,EAAA,QAAmB,KAMtB,SAAA,UAAA,SAAP,SAAgB,EAAW,EAAiB,CAC1C,GADyB,IAAA,QAAA,GAAA,GACrB,KAAK,OACP,MAAO,MAIT,KAAK,MAAQ,EAEb,GAAM,GAAK,KAAK,GACV,EAAY,KAAK,UAuBvB,MAAI,IAAM,MACR,MAAK,GAAK,KAAK,eAAe,EAAW,EAAI,IAK/C,KAAK,QAAU,GAEf,KAAK,MAAQ,EAEb,KAAK,GAAK,KAAK,IAAM,KAAK,eAAe,EAAW,KAAK,GAAI,GAEtD,MAGC,EAAA,UAAA,eAAV,SAAyB,EAA2B,EAAW,EAAiB,CAAjB,MAAA,KAAA,QAAA,GAAA,GACtD,GAAiB,YAAY,EAAU,MAAM,KAAK,EAAW,MAAO,IAGnE,EAAA,UAAA,eAAV,SAAyB,EAA4B,EAAS,EAAwB,CAEpF,GAF4D,IAAA,QAAA,GAAA,GAExD,GAAS,MAAQ,KAAK,QAAU,GAAS,KAAK,UAAY,GAC5D,MAAO,GAIT,GAAiB,cAAc,IAQ1B,EAAA,UAAA,QAAP,SAAe,EAAU,EAAa,CACpC,GAAI,KAAK,OACP,MAAO,IAAI,OAAM,gCAGnB,KAAK,QAAU,GACf,GAAM,GAAQ,KAAK,SAAS,EAAO,GACnC,GAAI,EACF,MAAO,GACF,AAAI,KAAK,UAAY,IAAS,KAAK,IAAM,MAc9C,MAAK,GAAK,KAAK,eAAe,KAAK,UAAW,KAAK,GAAI,QAIjD,EAAA,UAAA,SAAV,SAAmB,EAAU,EAAc,CACzC,GAAI,GAAmB,GACnB,EACJ,GAAI,CACF,KAAK,KAAK,SACH,EAAP,CACA,EAAU,GACV,EAAc,CAAC,CAAC,GAAK,GAAM,GAAI,OAAM,GAEvC,GAAI,EACF,YAAK,cACE,GAIX,EAAA,UAAA,YAAA,UAAA,CACE,GAAI,CAAC,KAAK,OAAQ,CACV,GAAA,GAAoB,KAAlB,EAAE,EAAA,GAAE,EAAS,EAAA,UACb,EAAY,EAAS,QAE7B,KAAK,KAAO,KAAK,MAAQ,KAAK,UAAY,KAC1C,KAAK,QAAU,GAEf,GAAU,EAAS,MACf,GAAM,MACR,MAAK,GAAK,KAAK,eAAe,EAAW,EAAI,OAG/C,KAAK,MAAQ,KACb,EAAA,UAAM,YAAW,KAAA,QAGvB,GAxIoC,ICiBpC,GAAA,IAAA,UAAA,CAGE,WAAoB,EAAoC,EAAiC,CAAjC,AAAA,IAAA,QAAA,GAAoB,EAAU,KAAlE,KAAA,oBAAA,EAClB,KAAK,IAAM,EA8BN,SAAA,UAAA,SAAP,SAAmB,EAAqD,EAAmB,EAAS,CAA5B,MAAA,KAAA,QAAA,GAAA,GAC/D,GAAI,MAAK,oBAAuB,KAAM,GAAM,SAAS,EAAO,IAlCvD,EAAA,IAAoB,GAAsB,IAoC1D,KCzDA,GAAA,IAAA,SAAA,EAAA,CAAoC,EAAA,EAAA,GAkBlC,WAAY,EAAgC,EAAiC,CAAjC,AAAA,IAAA,QAAA,GAAoB,GAAU,KAA1E,GAAA,GACE,EAAA,KAAA,KAAM,EAAiB,IAAI,KAlBtB,SAAA,QAAmC,GAOnC,EAAA,QAAmB,GAQnB,EAAA,WAAkB,SAMlB,SAAA,UAAA,MAAP,SAAa,EAAwB,CAC3B,GAAA,GAAY,KAAI,QAExB,GAAI,KAAK,QAAS,CAChB,EAAQ,KAAK,GACb,OAGF,GAAI,GACJ,KAAK,QAAU,GAEf,EACE,IAAK,EAAQ,EAAO,QAAQ,EAAO,MAAO,EAAO,OAC/C,YAEM,EAAS,EAAQ,SAI3B,GAFA,KAAK,QAAU,GAEX,EAAO,CACT,KAAQ,EAAS,EAAQ,SACvB,EAAO,cAET,KAAM,KAGZ,GAhDoC,IC8C7B,GAAM,IAAiB,GAAI,IAAe,IAKpC,GAAQ,GClDrB,GAAA,IAAA,SAAA,EAAA,CAA6C,EAAA,EAAA,GAC3C,WAAsB,EAA8C,EAAmD,CAAvH,GAAA,GACE,EAAA,KAAA,KAAM,EAAW,IAAK,KADF,SAAA,UAAA,EAA8C,EAAA,KAAA,IAI1D,SAAA,UAAA,eAAV,SAAyB,EAAoC,EAAU,EAAiB,CAEtF,MAFqE,KAAA,QAAA,GAAA,GAEjE,IAAU,MAAQ,EAAQ,EACrB,EAAA,UAAM,eAAc,KAAA,KAAC,EAAW,EAAI,GAG7C,GAAU,QAAQ,KAAK,MAIhB,EAAU,YAAe,GAAU,WAAa,GAAuB,sBAAsB,UAAA,CAAM,MAAA,GAAU,MAAM,aAElH,EAAA,UAAA,eAAV,SAAyB,EAAoC,EAAU,EAAiB,CAItF,GAJqE,IAAA,QAAA,GAAA,GAIhE,GAAS,MAAQ,EAAQ,GAAO,GAAS,MAAQ,KAAK,MAAQ,EACjE,MAAO,GAAA,UAAM,eAAc,KAAA,KAAC,EAAW,EAAI,GAK7C,AAAI,EAAU,QAAQ,SAAW,GAC/B,IAAuB,qBAAqB,GAC5C,EAAU,WAAa,SAK7B,GAlC6C,ICF7C,GAAA,IAAA,SAAA,EAAA,CAA6C,EAAA,EAAA,GAA7C,YAAA,gDACS,SAAA,UAAA,MAAP,SAAa,EAAyB,CACpC,KAAK,QAAU,GACf,KAAK,WAAa,OAEV,GAAA,GAAY,KAAI,QACpB,EACA,EAAQ,GACZ,EAAS,GAAU,EAAQ,QAC3B,GAAM,GAAQ,EAAQ,OAEtB,EACE,IAAK,EAAQ,EAAO,QAAQ,EAAO,MAAO,EAAO,OAC/C,YAEK,EAAE,EAAQ,GAAU,GAAS,EAAQ,UAI9C,GAFA,KAAK,QAAU,GAEX,EAAO,CACT,KAAO,EAAE,EAAQ,GAAU,GAAS,EAAQ,UAC1C,EAAO,cAET,KAAM,KAGZ,GA1B6C,ICgCtC,GAAM,GAA0B,GAAI,IAAwB,ICR5D,GAAM,IAAQ,GAAI,GAAkB,SAAC,EAAU,CAAK,MAAA,GAAW,aCxBhE,YAA2B,EAAqB,EAAwB,CAC5E,MAAO,IAAI,GAAc,SAAC,EAAU,CAElC,GAAI,GAAI,EAER,MAAO,GAAU,SAAS,UAAA,CACxB,AAAI,IAAM,EAAM,OAGd,EAAW,WAIX,GAAW,KAAK,EAAM,MAIjB,EAAW,QACd,KAAK,gBCrBR,GAAM,IAAe,SAAI,EAAM,CAAwB,MAAA,IAAK,MAAO,GAAE,QAAW,UAAY,MAAO,IAAM,YCM1G,YAAoB,EAAU,CAClC,MAAO,GAAW,GAAK,KAAA,OAAL,EAAO,MCFrB,YAAgC,EAA6B,EAAwB,CACzF,MAAO,IAAI,GAAc,SAAA,EAAU,CACjC,GAAM,GAAM,GAAI,IAChB,SAAI,IAAI,EAAU,SAAS,UAAA,CACzB,GAAM,GAA+B,EAAc,MACnD,EAAI,IAAI,EAAW,UAAU,CAC3B,KAAI,SAAC,EAAK,CAAI,EAAI,IAAI,EAAU,SAAS,UAAA,CAAM,MAAA,GAAW,KAAK,OAC/D,MAAK,SAAC,EAAG,CAAI,EAAI,IAAI,EAAU,SAAS,UAAA,CAAM,MAAA,GAAW,MAAM,OAC/D,SAAQ,UAAA,CAAK,EAAI,IAAI,EAAU,SAAS,UAAA,CAAM,MAAA,GAAW,qBAGtD,ICbL,YAA6B,EAAuB,EAAwB,CAChF,MAAO,IAAI,GAAc,SAAC,EAAU,CAClC,MAAO,GAAU,SAAS,UAAA,CACxB,MAAA,GAAM,KACJ,SAAC,EAAK,CACJ,EAAW,IACT,EAAU,SAAS,UAAA,CACjB,EAAW,KAAK,GAChB,EAAW,IAAI,EAAU,SAAS,UAAA,CAAM,MAAA,GAAW,kBAIzD,SAAC,EAAG,CACF,EAAW,IAAI,EAAU,SAAS,UAAA,CAAM,MAAA,GAAW,MAAM,YChB7D,aAA2B,CAC/B,MAAI,OAAO,SAAW,YAAc,CAAC,OAAO,SACnC,aAGF,OAAO,SAGT,GAAM,IAAW,KCJlB,YACJ,EACA,EACA,EACA,EAAS,CAAT,AAAA,IAAA,QAAA,GAAA,GAEA,GAAM,GAAe,EAAU,SAAS,UAAA,CACtC,GAAI,CACF,EAAQ,KAAK,YACN,EAAP,CACA,EAAW,MAAM,KAElB,GACH,SAAW,IAAI,GACR,ECPH,YAA8B,EAAoB,EAAwB,CAC9E,MAAO,IAAI,GAAc,SAAC,EAAU,CAClC,GAAI,GAKJ,SAAW,IACT,EAAU,SAAS,UAAA,CAEjB,EAAY,EAAc,MAG1B,GAAe,EAAY,EAAW,UAAA,CAE9B,GAAA,GAAkB,EAAS,OAAzB,EAAK,EAAA,MAAE,EAAI,EAAA,KACnB,AAAI,EAKF,EAAW,WAGX,GAAW,KAAK,GAGhB,KAAK,iBAUN,UAAA,CAAM,MAAA,GAAW,GAAQ,KAAA,OAAR,EAAU,SAAW,EAAS,YC5CpD,YAAmC,EAAyB,EAAwB,CACxF,GAAI,CAAC,EACH,KAAM,IAAI,OAAM,2BAElB,MAAO,IAAI,GAAc,SAAA,EAAU,CACjC,GAAM,GAAM,GAAI,IAChB,SAAI,IACF,EAAU,SAAS,UAAA,CACjB,GAAM,GAAW,EAAM,OAAO,iBAC9B,EAAI,IAAI,EAAU,SAAS,UAAA,CAAA,GAAA,GAAA,KACzB,EAAS,OAAO,KAAK,SAAA,EAAM,CACzB,AAAI,EAAO,KACT,EAAW,WAEX,GAAW,KAAK,EAAO,OACvB,EAAK,oBAMR,ICpBL,YAA8B,EAAU,CAC5C,MAAO,GAAW,EAAM,KCFpB,YAAqB,EAAU,CACnC,MAAO,GAAW,GAAK,KAAA,OAAL,EAAQ,KCHtB,YAA6B,EAAQ,CACzC,MAAO,QAAO,eAAiB,EAAW,GAAG,KAAA,OAAH,EAAM,OAAO,gBCCnD,YAA2C,EAAU,CAEzD,MAAO,IAAI,WACT,gBACE,KAAU,MAAQ,MAAO,IAAU,SAAW,oBAAsB,IAAI,EAAK,KAAG,4HCLhF,YAAuD,EAAqC,mGAC1F,EAAS,EAAe,qEAGF,MAAA,CAAA,EAAA,GAAM,EAAO,sBAA/B,GAAkB,EAAA,OAAhB,EAAK,EAAA,MAAE,EAAI,EAAA,KACf,iBAAA,CAAA,EAAA,UACF,MAAA,CAAA,EAAA,EAAA,2BAEI,WAAN,MAAA,CAAA,EAAA,EAAA,eAAA,SAAA,wCAGF,SAAO,yCAIL,YAAkC,EAAQ,CAG9C,MAAO,GAAW,GAAG,KAAA,OAAH,EAAK,WChBnB,YAAwC,EAA8B,EAAwB,CAClG,MAAO,IAAsB,GAAmC,GAAQ,GCqBpE,YAAuB,EAA2B,EAAwB,CAC9E,GAAI,GAAS,KAAM,CACjB,GAAI,GAAoB,GACtB,MAAO,IAAmB,EAAO,GAEnC,GAAI,GAAY,GACd,MAAO,IAAc,EAAO,GAE9B,GAAI,GAAU,GACZ,MAAO,IAAgB,EAAO,GAEhC,GAAI,GAAgB,GAClB,MAAO,IAAsB,EAAO,GAEtC,GAAI,GAAW,GACb,MAAO,IAAiB,EAAO,GAEjC,GAAI,GAAqB,GACvB,MAAO,IAA2B,EAAO,GAG7C,KAAM,IAAiC,GCqEnC,YAAkB,EAA2B,EAAyB,CAC1E,MAAO,GAAY,GAAU,EAAO,GAAa,EAAU,GAMvD,WAAuB,EAAyB,CACpD,GAAI,YAAiB,GACnB,MAAO,GAET,GAAI,GAAS,KAAM,CACjB,GAAI,GAAoB,GACtB,MAAO,IAAsB,GAE/B,GAAI,GAAY,GACd,MAAO,IAAc,GAEvB,GAAI,GAAU,GACZ,MAAO,IAAY,GAErB,GAAI,GAAgB,GAClB,MAAO,IAAkB,GAE3B,GAAI,GAAW,GACb,MAAO,IAAa,GAEtB,GAAI,GAAqB,GACvB,MAAO,IAAuB,GAIlC,KAAM,IAAiC,GAOzC,YAAkC,EAAQ,CACxC,MAAO,IAAI,GAAW,SAAC,EAAyB,CAC9C,GAAM,GAAM,EAAI,MAChB,GAAI,EAAW,EAAI,WACjB,MAAO,GAAI,UAAU,GAGvB,KAAM,IAAI,WAAU,oEAWlB,YAA2B,EAAmB,CAClD,MAAO,IAAI,GAAW,SAAC,EAAyB,CAU9C,OAAS,GAAI,EAAG,EAAI,EAAM,QAAU,CAAC,EAAW,OAAQ,IACtD,EAAW,KAAK,EAAM,IAExB,EAAW,aAIf,YAAwB,EAAuB,CAC7C,MAAO,IAAI,GAAW,SAAC,EAAyB,CAC9C,EACG,KACC,SAAC,EAAK,CACJ,AAAK,EAAW,QACd,GAAW,KAAK,GAChB,EAAW,aAGf,SAAC,EAAQ,CAAK,MAAA,GAAW,MAAM,KAEhC,KAAK,KAAM,MAIlB,YAAyB,EAAqB,CAC5C,MAAO,IAAI,GAAW,SAAC,EAAyB,aAC9C,OAAoB,GAAA,GAAA,GAAQ,EAAA,EAAA,OAAA,CAAA,EAAA,KAAA,EAAA,EAAA,OAAE,CAAzB,GAAM,GAAK,EAAA,MAEd,GADA,EAAW,KAAK,GACZ,EAAW,OACb,yGAGJ,EAAW,aAIf,YAA8B,EAA+B,CAC3D,MAAO,IAAI,GAAW,SAAC,EAAyB,CAC9C,GAAQ,EAAe,GAAY,MAAM,SAAC,EAAG,CAAK,MAAA,GAAW,MAAM,OAIvE,YAAmC,EAAqC,CACtE,MAAO,IAAkB,GAAmC,IAG9D,YAA0B,EAAiC,EAAyB,uIACxD,EAAA,GAAA,iFAIxB,GAJe,EAAK,EAAA,MACpB,EAAW,KAAK,GAGZ,EAAW,OACb,MAAA,CAAA,8RAGJ,SAAW,oBC3OP,YAA+B,EAAqB,EAAyB,CACjF,MAAO,GAAY,GAAc,EAAO,GAAa,GAAc,GCF/D,YAAsB,EAAU,CACpC,MAAO,IAAS,EAAW,EAAM,UCAnC,YAAiB,EAAQ,CACvB,MAAO,GAAI,EAAI,OAAS,GAGpB,YAA4B,EAAW,CAC3C,MAAO,GAAW,GAAK,IAAS,EAAK,MAAQ,OAGzC,YAAuB,EAAW,CACtC,MAAO,IAAY,GAAK,IAAS,EAAK,MAAQ,OAG1C,YAAoB,EAAa,EAAoB,CACzD,MAAO,OAAO,IAAK,IAAU,SAAW,EAAK,MAAS,EC+DlD,YAAY,QAAI,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GACpB,GAAM,GAAY,GAAa,GAC/B,MAAO,GAAY,GAAc,EAAa,GAAa,GAAkB,GC3EzE,YAAsB,EAAU,CACpC,MAAO,aAAiB,OAAQ,CAAC,MAAM,GCqCnC,WAAoB,EAAyC,EAAa,CAC9E,MAAO,GAAQ,SAAC,EAAQ,EAAU,CAEhC,GAAI,GAAQ,EAGZ,EAAO,UACL,GAAI,GAAmB,EAAY,SAAC,EAAQ,CAG1C,EAAW,KAAK,EAAQ,KAAK,EAAS,EAAO,WCpD7C,GAAA,IAAY,MAAK,QAEzB,YAA2B,EAA6B,EAAW,CAC/D,MAAO,IAAQ,GAAQ,EAAE,MAAA,OAAA,EAAA,GAAA,EAAI,KAAQ,EAAG,GAOtC,YAAiC,EAA2B,CAC9D,MAAO,GAAI,SAAA,EAAI,CAAI,MAAA,IAAY,EAAI,KC0CjC,WAAuB,EAA0B,EAAiB,CAAjB,MAAA,KAAA,QAAA,GAAA,GAC9C,EAAQ,SAAC,EAAQ,EAAU,CAChC,EAAO,UACL,GAAI,GACF,EACA,SAAC,EAAK,CAAK,MAAA,GAAW,IAAI,EAAU,SAAS,UAAA,CAAM,MAAA,GAAW,KAAK,IAAQ,KAC3E,UAAA,CAAM,MAAA,GAAW,IAAI,EAAU,SAAS,UAAA,CAAM,MAAA,GAAW,YAAY,KACrE,SAAC,EAAG,CAAK,MAAA,GAAW,IAAI,EAAU,SAAS,UAAA,CAAM,MAAA,GAAW,MAAM,IAAM,SC/DxE,GAAA,IAAY,MAAK,QACjB,GAA0D,OAAM,eAArC,GAA+B,OAAM,UAAlB,GAAY,OAAM,KAQlE,YAA+D,EAAuB,CAC1F,GAAI,EAAK,SAAW,EAAG,CACrB,GAAM,GAAQ,EAAK,GACnB,GAAI,GAAQ,GACV,MAAO,CAAE,KAAM,EAAO,KAAM,MAE9B,GAAI,GAAO,GAAQ,CACjB,GAAM,GAAO,GAAQ,GACrB,MAAO,CACL,KAAM,EAAK,IAAI,SAAC,EAAG,CAAK,MAAA,GAAM,KAC9B,KAAI,IAKV,MAAO,CAAE,KAAM,EAAa,KAAM,MAGpC,YAAgB,EAAQ,CACtB,MAAO,IAAO,MAAO,IAAQ,UAAY,GAAe,KAAS,GC5B7D,YAAuB,EAAgB,EAAa,CACxD,MAAO,GAAK,OAAO,SAAC,EAAQ,EAAK,EAAC,CAAK,MAAE,GAAO,GAAO,EAAO,GAAK,GAAS,ICmMxE,YAAuB,QAAoC,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GAC/D,GAAM,GAAY,GAAa,GACzB,EAAiB,GAAkB,GAEnC,EAA8B,GAAqB,GAA3C,EAAW,EAAA,KAAE,EAAI,EAAA,KAE/B,GAAI,EAAY,SAAW,EAIzB,MAAO,IAAK,GAAI,GAGlB,GAAM,GAAS,GAAI,GACjB,GACE,EACA,EACA,EAEI,SAAC,EAAM,CAAK,MAAA,IAAa,EAAM,IAE/B,KAIR,MAAO,GAAkB,EAAO,KAAK,GAAiB,IAAqC,EAGvF,YACJ,EACA,EACA,EAAiD,CAAjD,MAAA,KAAA,QAAA,GAAA,IAEO,SAAC,EAA2B,CAGjC,GACE,EACA,UAAA,CAaE,OAZQ,GAAW,EAAW,OAExB,EAAS,GAAI,OAAM,GAGrB,EAAS,EAIT,EAAuB,aAGlB,EAAC,CACR,GACE,EACA,UAAA,CACE,GAAM,GAAS,GAAK,EAAY,GAAI,GAChC,EAAgB,GACpB,EAAO,UACL,GAAI,GACF,EACA,SAAC,EAAK,CAEJ,EAAO,GAAK,EACP,GAEH,GAAgB,GAChB,KAEG,GAGH,EAAW,KAAK,EAAe,EAAO,WAG1C,UAAA,CACE,AAAK,EAAE,GAGL,EAAW,eAMrB,IAjCK,EAAI,EAAG,EAAI,EAAQ,MAAnB,IAqCX,IASN,YAAuB,EAAsC,EAAqB,EAA0B,CAC1G,AAAI,EACF,EAAa,IAAI,EAAU,SAAS,IAEpC,ICtRE,YACJ,EACA,EACA,EACA,EACA,EACA,EACA,EACA,EAA+B,CAG/B,GAAM,GAAc,GAEhB,EAAS,EAET,EAAQ,EAER,EAAa,GAKX,EAAgB,UAAA,CAIpB,AAAI,GAAc,CAAC,EAAO,QAAU,CAAC,GACnC,EAAW,YAKT,EAAY,SAAC,EAAQ,CAAK,MAAC,GAAS,EAAa,EAAW,GAAS,EAAO,KAAK,IAEjF,EAAa,SAAC,EAAQ,CAI1B,GAAU,EAAW,KAAK,GAI1B,IAKA,GAAI,GAAgB,GAGpB,EAAU,EAAQ,EAAO,MAAU,UACjC,GAAI,GACF,EACA,SAAC,EAAU,CAGT,GAAY,MAAZ,EAAe,GAEf,AAAI,EAGF,EAAU,GAGV,EAAW,KAAK,IAGpB,UAAA,CAGE,EAAgB,IAGlB,OACA,UAAA,CAIE,GAAI,EAKF,GAAI,CAIF,IAKA,qBACE,GAAM,GAAgB,EAAO,QAI7B,EAAoB,EAAW,IAAI,EAAkB,SAAS,UAAA,CAAM,MAAA,GAAW,MAAmB,EAAW,IALxG,EAAO,QAAU,EAAS,OAQjC,UACO,EAAP,CACA,EAAW,MAAM,QAS7B,SAAO,UACL,GAAI,GAAmB,EAAY,EAAW,UAAA,CAE5C,EAAa,GACb,OAMG,UAAA,CACL,GAAkB,MAAlB,KC7DE,YACJ,EACA,EACA,EAA6B,CAE7B,MAFA,KAAA,QAAA,GAAA,KAEI,EAAW,GAEN,GAAS,SAAC,EAAG,EAAC,CAAK,MAAA,GAAI,SAAC,EAAQ,EAAU,CAAK,MAAA,GAAe,EAAG,EAAG,EAAG,KAAK,EAAU,EAAQ,EAAG,MAAM,GACrG,OAAO,IAAmB,UACnC,GAAa,GAGR,EAAQ,SAAC,EAAQ,EAAU,CAAK,MAAA,IAAe,EAAQ,EAAY,EAAS,MChC/E,YAAmD,EAA6B,CAA7B,MAAA,KAAA,QAAA,GAAA,KAChD,GAAS,GAAU,GCFtB,aAAmB,CACvB,MAAO,IAAS,GCsDZ,aAAgB,QAAC,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GACrB,MAAO,MAAY,GAAkB,EAAM,GAAa,KCjEpD,YAAgD,EAA0B,CAC9E,MAAO,IAAI,GAA+B,SAAC,EAAU,CACnD,EAAU,KAAqB,UAAU,KC5C7C,GAAM,IAA0B,CAAC,cAAe,kBAC1C,GAAqB,CAAC,mBAAoB,uBAC1C,GAAgB,CAAC,KAAM,OA2NvB,WACJ,EACA,EACA,EACA,EAAsC,CAMtC,GAJI,EAAW,IACb,GAAiB,EACjB,EAAU,QAER,EACF,MAAO,GAAa,EAAQ,EAAW,GAAiC,KAAK,GAAiB,IAU1F,GAAA,GAAA,EAEJ,GAAc,GACV,GAAmB,IAAI,SAAC,EAAU,CAAK,MAAA,UAAC,EAAY,CAAK,MAAA,GAAO,GAAY,EAAW,EAAS,MAElG,GAAwB,GACtB,GAAwB,IAAI,GAAwB,EAAQ,IAC5D,GAA0B,GAC1B,GAAc,IAAI,GAAwB,EAAQ,IAClD,GAAE,GATD,EAAG,EAAA,GAAE,EAAM,EAAA,GAgBlB,GAAI,CAAC,GACC,GAAY,GACd,MAAO,IAAS,SAAC,EAAc,CAAK,MAAA,GAAU,EAAW,EAAW,KAClE,GAAkB,IAOxB,GAAI,CAAC,EACH,KAAM,IAAI,WAAU,wBAGtB,MAAO,IAAI,GAAc,SAAC,EAAU,CAIlC,GAAM,GAAU,UAAA,QAAC,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GAAmB,MAAA,GAAW,KAAK,EAAI,EAAK,OAAS,EAAO,EAAK,KAElF,SAAI,GAEG,UAAA,CAAM,MAAA,GAAQ,MAWzB,YAAiC,EAAa,EAAiB,CAC7D,MAAO,UAAC,EAAkB,CAAK,MAAA,UAAC,EAAY,CAAK,MAAA,GAAO,GAAY,EAAW,KAQjF,YAAiC,EAAW,CAC1C,MAAO,GAAW,EAAO,cAAgB,EAAW,EAAO,gBAQ7D,YAAmC,EAAW,CAC5C,MAAO,GAAW,EAAO,KAAO,EAAW,EAAO,KAQpD,YAAuB,EAAW,CAChC,MAAO,GAAW,EAAO,mBAAqB,EAAW,EAAO,qBC1L5D,YACJ,EACA,EACA,EAAsC,CAEtC,MAAI,GACK,GAAoB,EAAY,GAAe,KAAK,GAAiB,IAGvE,GAAI,GAAoB,SAAC,EAAU,CACxC,GAAM,GAAU,UAAA,QAAC,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GAAc,MAAA,GAAW,KAAK,EAAE,SAAW,EAAI,EAAE,GAAK,IACjE,EAAW,EAAW,GAC5B,MAAO,GAAW,GAAiB,UAAA,CAAM,MAAA,GAAc,EAAS,IAAY,SClB1E,YACJ,EACA,EACA,EAAyC,CAFzC,AAAA,IAAA,QAAA,GAAA,GAEA,IAAA,QAAA,GAAA,IAIA,GAAI,GAAmB,GAEvB,MAAI,IAAuB,MAIzB,CAAI,GAAY,GACd,EAAY,EAIZ,EAAmB,GAIhB,GAAI,GAAW,SAAC,EAAU,CAI/B,GAAI,GAAM,GAAY,GAAW,CAAC,EAAU,EAAW,MAAQ,EAE/D,AAAI,EAAM,GAER,GAAM,GAIR,GAAI,GAAI,EAGR,MAAO,GAAU,SAAS,UAAA,CACxB,AAAK,EAAW,QAEd,GAAW,KAAK,KAEhB,AAAI,GAAK,EAGP,KAAK,SAAS,OAAW,GAGzB,EAAW,aAGd,KCpGD,YAAe,QAAC,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GACpB,GAAM,GAAY,GAAa,GACzB,EAAa,GAAU,EAAM,KAC7B,EAAU,EAChB,MAAO,AAAC,GAAQ,OAGZ,EAAQ,SAAW,EAEnB,EAAU,EAAQ,IAElB,GAAS,GAAY,GAAkB,EAAS,IALhD,GC3DC,GAAM,GAAQ,GAAI,GAAkB,GCjCnC,GAAA,IAAY,MAAK,QAMnB,YAA4B,EAAiB,CACjD,MAAO,GAAK,SAAW,GAAK,GAAQ,EAAK,IAAM,EAAK,GAAM,ECgDtD,WAAoB,EAAiD,EAAa,CACtF,MAAO,GAAQ,SAAC,EAAQ,EAAU,CAEhC,GAAI,GAAQ,EAIZ,EAAO,UAIL,GAAI,GAAmB,EAAY,SAAC,EAAK,CAAK,MAAA,GAAU,KAAK,EAAS,EAAO,MAAY,EAAW,KAAK,QChBzG,aAAa,QAAC,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GAClB,GAAM,GAAiB,GAAkB,GAEnC,EAAU,GAAe,GAE/B,MAAO,GAAQ,OACX,GAAI,GAAsB,SAAC,EAAU,CAGnC,GAAI,GAAuB,EAAQ,IAAI,UAAA,CAAM,MAAA,KAKzC,EAAY,EAAQ,IAAI,UAAA,CAAM,MAAA,KAGlC,EAAW,IAAI,UAAA,CACb,EAAU,EAAY,OAMxB,mBAAS,EAAW,CAClB,EAAU,EAAQ,IAAc,UAC9B,GAAI,GACF,EACA,SAAC,EAAK,CAKJ,GAJA,EAAQ,GAAa,KAAK,GAItB,EAAQ,MAAM,SAAC,EAAM,CAAK,MAAA,GAAO,SAAS,CAC5C,GAAM,GAAc,EAAQ,IAAI,SAAC,EAAM,CAAK,MAAA,GAAO,UAEnD,EAAW,KAAK,EAAiB,EAAc,MAAA,OAAA,EAAA,GAAA,EAAI,KAAU,GAIzD,EAAQ,KAAK,SAAC,EAAQ,EAAC,CAAK,MAAA,CAAC,EAAO,QAAU,EAAU,MAC1D,EAAW,aAIjB,UAAA,CAGE,EAAU,GAAe,GAIzB,CAAC,EAAQ,GAAa,QAAU,EAAW,eA5B1C,EAAc,EAAG,CAAC,EAAW,QAAU,EAAc,EAAQ,OAAQ,MAArE,GAmCT,MAAO,WAAA,CACL,EAAU,EAAY,QAG1B,GCvDA,YAAyB,EAAoB,EAAsC,CAAtC,MAAA,KAAA,QAAA,GAAA,MAGjD,EAAmB,GAAgB,KAAhB,EAAoB,EAEhC,EAAQ,SAAC,EAAQ,EAAU,CAChC,GAAI,GAAiB,GACjB,EAAQ,EAEZ,EAAO,UACL,GAAI,GACF,EACA,SAAC,EAAK,aACA,EAAuB,KAK3B,AAAI,IAAU,GAAsB,GAClC,EAAQ,KAAK,QAIf,OAAqB,GAAA,GAAA,GAAO,EAAA,EAAA,OAAA,CAAA,EAAA,KAAA,EAAA,EAAA,OAAE,CAAzB,GAAM,GAAM,EAAA,MACf,EAAO,KAAK,GAMR,GAAc,EAAO,QACvB,GAAS,GAAM,KAAN,EAAU,GACnB,EAAO,KAAK,sGAIhB,GAAI,MAIF,OAAqB,GAAA,GAAA,GAAM,EAAA,EAAA,OAAA,CAAA,EAAA,KAAA,EAAA,EAAA,OAAE,CAAxB,GAAM,GAAM,EAAA,MACf,GAAU,EAAS,GACnB,EAAW,KAAK,uGAItB,UAAA,aAGE,OAAqB,GAAA,GAAA,GAAO,EAAA,EAAA,OAAA,CAAA,EAAA,KAAA,EAAA,EAAA,OAAE,CAAzB,GAAM,GAAM,EAAA,MACf,EAAW,KAAK,qGAElB,EAAW,YAGb,OACA,UAAA,CAEE,EAAU,UCXd,YACJ,EAAgD,CAEhD,MAAO,GAAQ,SAAC,EAAQ,EAAU,CAChC,GAAI,GAAgC,KAChC,EAAY,GACZ,EAEJ,EAAW,EAAO,UAChB,GAAI,GAAmB,EAAY,OAAW,OAAW,SAAC,EAAG,CAC3D,EAAgB,EAAU,EAAS,EAAK,GAAW,GAAU,KAC7D,AAAI,EACF,GAAS,cACT,EAAW,KACX,EAAc,UAAU,IAIxB,EAAY,MAKd,GAMF,GAAS,cACT,EAAW,KACX,EAAe,UAAU,MC3HzB,YACJ,EACA,EACA,EACA,EACA,EAAqC,CAErC,MAAO,UAAC,EAAuB,EAA2B,CAIxD,GAAI,GAAW,EAIX,EAAa,EAEb,EAAQ,EAGZ,EAAO,UACL,GAAI,GACF,EACA,SAAC,EAAK,CAEJ,GAAM,GAAI,IAEV,EAAQ,EAEJ,EAAY,EAAO,EAAO,GAIxB,GAAW,GAAO,GAGxB,GAAc,EAAW,KAAK,IAIhC,GACG,UAAA,CACC,GAAY,EAAW,KAAK,GAC5B,EAAW,eC9BjB,aAAuB,QAAO,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GAClC,GAAM,GAAiB,GAAkB,GACzC,MAAO,GACH,GAAK,GAAa,MAAA,OAAA,EAAA,GAAA,EAAK,KAAuC,GAAiB,IAC/E,EAAQ,SAAC,EAAQ,EAAU,CACzB,GAAiB,EAAA,CAAE,GAAM,EAAK,GAAe,MAAQ,KCUvD,aAA2B,QAC/B,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GAEA,MAAO,IAAa,MAAA,OAAA,EAAA,GAAA,EAAI,KCkCpB,YACJ,EACA,EAA6G,CAE7G,MAAO,GAAW,GAAkB,GAAS,EAAS,EAAgB,GAAK,GAAS,EAAS,GCnBzF,YAA0B,EAAiB,EAAyC,CAAzC,MAAA,KAAA,QAAA,GAAA,IACxC,EAAQ,SAAC,EAAQ,EAAU,CAChC,GAAI,GAAkC,KAClC,EAAsB,KACtB,EAA0B,KAExB,EAAO,UAAA,CACX,GAAI,EAAY,CAEd,EAAW,cACX,EAAa,KACb,GAAM,GAAQ,EACd,EAAY,KACZ,EAAW,KAAK,KAGpB,YAAqB,CAInB,GAAM,GAAa,EAAY,EACzB,EAAM,EAAU,MACtB,GAAI,EAAM,EAAY,CAEpB,EAAa,KAAK,SAAS,OAAW,EAAa,GACnD,EAAW,IAAI,GACf,OAGF,IAGF,EAAO,UACL,GAAI,GACF,EACA,SAAC,EAAQ,CACP,EAAY,EACZ,EAAW,EAAU,MAGhB,GACH,GAAa,EAAU,SAAS,EAAc,GAC9C,EAAW,IAAI,KAGnB,UAAA,CAGE,IACA,EAAW,YAGb,OACA,UAAA,CAEE,EAAY,EAAa,UChF7B,YAA+B,EAAe,CAClD,MAAO,GAAQ,SAAC,EAAQ,EAAU,CAChC,GAAI,GAAW,GACf,EAAO,UACL,GAAI,GACF,EACA,SAAC,EAAK,CACJ,EAAW,GACX,EAAW,KAAK,IAElB,UAAA,CACE,AAAK,GACH,EAAW,KAAK,GAElB,EAAW,gBCNf,YAAkB,EAAa,CACnC,MAAO,IAAS,EAEZ,UAAA,CAAM,MAAA,KACN,EAAQ,SAAC,EAAQ,EAAU,CACzB,GAAI,GAAO,EACX,EAAO,UACL,GAAI,GAAmB,EAAY,SAAC,EAAK,CAIvC,AAAI,EAAE,GAAQ,GACZ,GAAW,KAAK,GAIZ,GAAS,GACX,EAAW,iBC1BrB,aAAwB,CAC5B,MAAO,GAAQ,SAAC,EAAQ,EAAU,CAChC,EAAO,UAAU,GAAI,GAAmB,EAAY,MCFlD,YAAmB,EAAQ,CAC/B,MAAO,GAAI,UAAA,CAAM,MAAA,KCmCb,YACJ,EACA,EAAmC,CAEnC,MAAI,GAEK,SAAC,EAAqB,CAC3B,MAAA,IAAO,EAAkB,KAAK,GAAK,GAAI,MAAmB,EAAO,KAAK,GAAU,MAG7E,GAAS,SAAC,EAAO,EAAK,CAAK,MAAA,GAAsB,EAAO,GAAO,KAAK,GAAK,GAAI,GAAM,MCvBtF,YAAmB,EAAoB,EAAyC,CAAzC,AAAA,IAAA,QAAA,GAAA,IAC3C,GAAM,GAAW,GAAM,EAAK,GAC5B,MAAO,IAAU,UAAA,CAAM,MAAA,KCoFnB,WACJ,EACA,EAA0D,CAA1D,MAAA,KAAA,QAAA,GAA+B,IAK/B,EAAa,GAAU,KAAV,EAAc,GAEpB,EAAQ,SAAC,EAAQ,EAAU,CAGhC,GAAI,GAEA,EAAQ,GAEZ,EAAO,UACL,GAAI,GAAmB,EAAY,SAAC,EAAK,CAEvC,GAAM,GAAa,EAAY,GAK/B,AAAI,IAAS,CAAC,EAAY,EAAa,KAMrC,GAAQ,GACR,EAAc,EAGd,EAAW,KAAK,SAO1B,YAAwB,EAAQ,EAAM,CACpC,MAAO,KAAM,EC/GT,WAAwD,EAAQ,EAAuC,CAC3G,MAAO,GAAqB,SAAC,EAAM,EAAI,CAAK,MAAA,GAAU,EAAQ,EAAE,GAAM,EAAE,IAAQ,EAAE,KAAS,EAAE,KCnBzF,YAAsB,EAAoB,CAC9C,MAAO,GAAQ,SAAC,EAAQ,EAAU,CAGhC,GAAI,CACF,EAAO,UAAU,WAEjB,EAAW,IAAI,MCrBf,YAAsB,EAAa,CACvC,MAAO,IAAS,EACZ,UAAA,CAAM,MAAA,KACN,EAAQ,SAAC,EAAQ,EAAU,CAKzB,GAAI,GAAc,GAClB,EAAO,UACL,GAAI,GACF,EACA,SAAC,EAAK,CAEJ,EAAO,KAAK,GAGZ,EAAQ,EAAO,QAAU,EAAO,SAElC,UAAA,aAGE,OAAoB,GAAA,GAAA,GAAM,EAAA,EAAA,OAAA,CAAA,EAAA,KAAA,EAAA,EAAA,OAAE,CAAvB,GAAM,GAAK,EAAA,MACd,EAAW,KAAK,qGAElB,EAAW,YAGb,OACA,UAAA,CAEE,EAAS,UCtDjB,aAAe,QAAI,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GACvB,GAAM,GAAY,GAAa,GACzB,EAAa,GAAU,EAAM,KACnC,SAAO,GAAe,GAEf,EAAQ,SAAC,EAAQ,EAAU,CAChC,GAAS,GAAY,GAAiB,EAAA,CAAE,GAAM,EAAM,IAAgC,IAAY,UAAU,KCgBxG,aAAmB,QACvB,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GAEA,MAAO,IAAK,MAAA,OAAA,EAAA,GAAA,EAAI,KCHZ,YAAoB,EAAyB,CACjD,MAAO,GAAQ,SAAC,EAAQ,EAAU,CAChC,GAAI,GAAW,GACX,EAAsB,KAC1B,EAAO,UACL,GAAI,GAAmB,EAAY,SAAC,EAAK,CACvC,EAAW,GACX,EAAY,KAGhB,GAAM,GAAO,UAAA,CACX,GAAI,EAAU,CACZ,EAAW,GACX,GAAM,GAAQ,EACd,EAAY,KACZ,EAAW,KAAK,KAGpB,EAAS,UAAU,GAAI,GAAmB,EAAY,EAAM,MC8B1D,YAAwB,EAA6D,EAAQ,CAMjG,MAAO,GAAQ,GAAc,EAAa,EAAW,UAAU,QAAU,EAAG,KCqCxE,YAAmB,EAA4B,CAA5B,AAAA,IAAA,QAAA,GAAA,IACf,GAAA,GAAgH,EAAO,UAAvH,EAAS,IAAA,OAAG,UAAA,CAAM,MAAA,IAAI,IAAY,EAAE,EAA4E,EAAO,aAAnF,EAAY,IAAA,OAAG,GAAI,EAAE,EAAuD,EAAO,gBAA9D,EAAe,IAAA,OAAG,GAAI,EAAE,EAA+B,EAAO,oBAAtC,EAAmB,IAAA,OAAG,GAAI,EAUnH,MAAO,UAAC,EAAa,CACnB,GAAI,GAAuC,KACvC,EAAuC,KACvC,EAAiC,KACjC,EAAW,EACX,EAAe,GACf,EAAa,GAEX,EAAc,UAAA,CAClB,GAAe,MAAf,EAAiB,cACjB,EAAkB,MAId,EAAQ,UAAA,CACZ,IACA,EAAa,EAAU,KACvB,EAAe,EAAa,IAExB,EAAsB,UAAA,CAG1B,GAAM,GAAO,EACb,IACA,GAAI,MAAJ,EAAM,eAGR,MAAO,GAAc,SAAC,EAAQ,GAAU,CACtC,IACI,CAAC,GAAc,CAAC,GAClB,IAOF,GAAM,IAAQ,EAAU,GAAO,KAAP,EAAW,IAOnC,GAAW,IAAI,UAAA,CACb,IAKI,IAAa,GAAK,CAAC,GAAc,CAAC,GACpC,GAAkB,GAAY,EAAqB,MAMvD,GAAK,UAAU,IAEV,GAMH,GAAa,GAAI,IAAe,CAC9B,KAAM,SAAC,GAAK,CAAK,MAAA,IAAK,KAAK,KAC3B,MAAO,SAAC,GAAG,CACT,EAAa,GACb,IACA,EAAkB,GAAY,EAAO,EAAc,IACnD,GAAK,MAAM,KAEb,SAAU,UAAA,CACR,EAAe,GACf,IACA,EAAkB,GAAY,EAAO,GACrC,GAAK,cAGT,GAAK,GAAQ,UAAU,MAExB,IAIP,YACE,EACA,EAA+C,QAC/C,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,EAAA,GAAA,UAAA,GAEA,MAAI,KAAO,GACT,KAEO,MAGL,IAAO,GACF,KAGF,EAAE,MAAA,OAAA,EAAA,GAAA,EAAI,KACV,KAAK,GAAK,IACV,UAAU,UAAA,CAAM,MAAA,OChIf,YACJ,EACA,EACA,EAAyB,SAErB,EACA,EAAW,GACf,MAAI,IAAsB,MAAO,IAAuB,SACtD,GAAa,GAAA,EAAmB,cAAU,MAAA,IAAA,OAAA,EAAI,IAC9C,EAAa,GAAA,EAAmB,cAAU,MAAA,IAAA,OAAA,EAAI,IAC9C,EAAW,CAAC,CAAC,EAAmB,SAChC,EAAY,EAAmB,WAE/B,EAAa,GAAkB,KAAlB,EAAsB,IAE9B,GAAS,CACd,UAAW,UAAA,CAAM,MAAA,IAAI,IAAc,EAAY,EAAY,IAC3D,aAAc,GACd,gBAAiB,GACjB,oBAAqB,IC1GnB,YAAkB,EAAa,CACnC,MAAO,GAAO,SAAC,EAAG,EAAK,CAAK,MAAA,IAAS,ICUjC,YAAuB,EAAyB,CACpD,MAAO,GAAQ,SAAC,EAAQ,EAAU,CAChC,GAAI,GAAS,GAEP,EAAiB,GAAI,GACzB,EACA,UAAA,CACE,GAAc,MAAd,EAAgB,cAChB,EAAS,IAEX,GAGF,EAAU,GAAU,UAAU,GAE9B,EAAO,UAAU,GAAI,GAAmB,EAAY,SAAC,EAAK,CAAK,MAAA,IAAU,EAAW,KAAK,QCDvF,YAAmB,QAAO,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GAC9B,GAAM,GAAY,GAAa,GAC/B,MAAO,GAAQ,SAAC,EAAQ,EAAU,CAIhC,AAAC,GAAY,GAAO,EAAQ,EAAQ,GAAa,GAAO,EAAQ,IAAS,UAAU,KCiBjF,WACJ,EACA,EAA6G,CAE7G,MAAO,GAAQ,SAAC,EAAQ,EAAU,CAChC,GAAI,GAAyD,KACzD,EAAQ,EAER,EAAa,GAIX,EAAgB,UAAA,CAAM,MAAA,IAAc,CAAC,GAAmB,EAAW,YAEzE,EAAO,UACL,GAAI,GACF,EACA,SAAC,EAAK,CAEJ,GAAe,MAAf,EAAiB,cACjB,GAAI,GAAa,EACX,EAAa,IAEnB,EAAU,EAAQ,EAAO,IAAa,UACnC,EAAkB,GAAI,GACrB,EAIA,SAAC,EAAU,CAAK,MAAA,GAAW,KAAK,EAAiB,EAAe,EAAO,EAAY,EAAY,KAAgB,IAC/G,UAAA,CAIE,EAAkB,KAClB,QAKR,UAAA,CACE,EAAa,GACb,SCnEJ,YACJ,EACA,EAA6G,CAE7G,MAAO,GAAW,GAAkB,EAAU,UAAA,CAAM,MAAA,IAAiB,GAAkB,EAAU,UAAA,CAAM,MAAA,KCjBnG,YAAuB,EAA8B,CACzD,MAAO,GAAQ,SAAC,EAAQ,EAAU,CAChC,EAAU,GAAU,UAAU,GAAI,GAAmB,EAAY,UAAA,CAAM,MAAA,GAAW,YAAY,IAC9F,CAAC,EAAW,QAAU,EAAO,UAAU,KCSrC,YAAuB,EAAiD,EAAiB,CAAjB,MAAA,KAAA,QAAA,GAAA,IACrE,EAAQ,SAAC,EAAQ,EAAU,CAChC,GAAI,GAAQ,EACZ,EAAO,UACL,GAAI,GAAmB,EAAY,SAAC,EAAK,CACvC,GAAM,GAAS,EAAU,EAAO,KAChC,AAAC,IAAU,IAAc,EAAW,KAAK,GACzC,CAAC,GAAU,EAAW,gBCkDxB,WACJ,EACA,EACA,EAA8B,CAK9B,GAAM,GACJ,EAAW,IAAmB,GAAS,EAElC,CAAE,KAAM,EAA2E,MAAK,EAAE,SAAQ,GACnG,EAEN,MAAO,GACH,EAAQ,SAAC,EAAQ,EAAU,OACzB,AAAA,GAAA,EAAY,aAAS,MAAA,IAAA,QAAA,EAAA,KAArB,GACA,GAAI,GAAU,GACd,EAAO,UACL,GAAI,GACF,EACA,SAAC,EAAK,OACJ,AAAA,GAAA,EAAY,QAAI,MAAA,IAAA,QAAA,EAAA,KAAhB,EAAmB,GACnB,EAAW,KAAK,IAElB,UAAA,OACE,EAAU,GACV,GAAA,EAAY,YAAQ,MAAA,IAAA,QAAA,EAAA,KAApB,GACA,EAAW,YAEb,SAAC,EAAG,OACF,EAAU,GACV,GAAA,EAAY,SAAK,MAAA,IAAA,QAAA,EAAA,KAAjB,EAAoB,GACpB,EAAW,MAAM,IAEnB,UAAA,SACE,AAAI,GACF,IAAA,EAAY,eAAW,MAAA,IAAA,QAAA,EAAA,KAAvB,IAEF,GAAA,EAAY,YAAQ,MAAA,IAAA,QAAA,EAAA,KAApB,QAQR,GCpJC,GAAM,IAAwC,CACnD,QAAS,GACT,SAAU,IA+CN,YACJ,EACA,EAA6D,IAA7D,GAAA,IAAA,OAAwC,GAAqB,EAA3D,EAAO,EAAA,QAAE,EAAQ,EAAA,SAEnB,MAAO,GAAQ,SAAC,EAAQ,EAAU,CAChC,GAAI,GAAW,GACX,EAAsB,KACtB,EAAiC,KACjC,EAAa,GAEX,EAAgB,UAAA,CACpB,GAAS,MAAT,EAAW,cACX,EAAY,KACR,GACF,KACA,GAAc,EAAW,aAIvB,EAAoB,UAAA,CACxB,EAAY,KACZ,GAAc,EAAW,YAGrB,EAAgB,SAAC,EAAQ,CAC7B,MAAC,GAAY,EAAU,EAAiB,IAAQ,UAAU,GAAI,GAAmB,EAAY,EAAe,KAExG,EAAO,UAAA,CACX,GAAI,EAAU,CAIZ,EAAW,GACX,GAAM,GAAQ,EACd,EAAY,KAEZ,EAAW,KAAK,GAChB,CAAC,GAAc,EAAc,KAIjC,EAAO,UACL,GAAI,GACF,EAMA,SAAC,EAAK,CACJ,EAAW,GACX,EAAY,EACZ,CAAE,IAAa,CAAC,EAAU,SAAY,GAAU,IAAS,EAAc,KAEzE,UAAA,CACE,EAAa,GACb,CAAE,IAAY,GAAY,GAAa,CAAC,EAAU,SAAW,EAAW,gBC7D5E,aAAwB,QAAO,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GACnC,GAAM,GAAU,GAAkB,GAElC,MAAO,GAAQ,SAAC,EAAQ,EAAU,CAehC,OAdM,GAAM,EAAO,OACb,EAAc,GAAI,OAAM,GAI1B,EAAW,EAAO,IAAI,UAAA,CAAM,MAAA,KAG5B,EAAQ,cAMH,EAAC,CACR,EAAU,EAAO,IAAI,UACnB,GAAI,GACF,EACA,SAAC,EAAK,CACJ,EAAY,GAAK,EACb,CAAC,GAAS,CAAC,EAAS,IAEtB,GAAS,GAAK,GAKb,GAAQ,EAAS,MAAM,MAAe,GAAW,QAKtD,KAlBG,EAAI,EAAG,EAAI,EAAK,MAAhB,GAwBT,EAAO,UACL,GAAI,GAAmB,EAAY,SAAC,EAAK,CACvC,GAAI,EAAO,CAET,GAAM,GAAM,EAAA,CAAI,GAAK,EAAK,IAC1B,EAAW,KAAK,EAAU,EAAO,MAAA,OAAA,EAAA,GAAA,EAAI,KAAU,SClFnD,aAAa,QAAO,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GACxB,MAAO,GAAQ,SAAC,EAAQ,EAAU,CAEhC,GAAS,MAAA,OAAA,EAAA,CAAC,GAAM,EAAM,KAAmB,UAAU,KCEjD,aAAiB,QAAkC,GAAA,GAAA,EAAA,EAAA,EAAA,UAAA,OAAA,IAAA,EAAA,GAAA,UAAA,GACvD,MAAO,IAAG,MAAA,OAAA,EAAA,GAAA,EAAI,KCUT,aAA4C,CACjD,GAAM,GAAY,GAAI,IACtB,SAAU,SAAU,oBACjB,KACC,GAAM,WAEL,UAAU,GAGR,ECFF,YACL,EAAkB,EAAmB,SACtB,CACf,MAAO,GAAK,cAAiB,IAAa,OAqBrC,YACL,EAAkB,EAAmB,SAClC,CACH,GAAM,GAAK,GAAc,EAAU,GACnC,GAAI,MAAO,IAAO,YAChB,KAAM,IAAI,gBACR,8BAA8B,oBAElC,MAAO,GAQF,aAAqD,CAC1D,MAAO,UAAS,wBAAyB,aACrC,SAAS,cACT,OAqBC,WACL,EAAkB,EAAmB,SAChC,CACL,MAAO,OAAM,KAAK,EAAK,iBAAoB,IActC,YACL,EAC0B,CAC1B,MAAO,UAAS,cAAc,GASzB,YACL,KAAoB,EACd,CACN,EAAG,YAAY,GAAG,GCvGb,YACL,EAAiB,EAAQ,GACnB,CACN,AAAI,EACF,EAAG,QAEH,EAAG,OAYA,YACL,EACqB,CACrB,MAAO,GACL,EAAsB,EAAI,SAC1B,EAAsB,EAAI,SAEzB,KACC,EAAI,CAAC,CAAE,UAAW,IAAS,SAC3B,EAAU,IAAO,OCNvB,GAAM,IAAS,GAAI,GAYb,GAAY,GAAM,IAAM,EAC5B,GAAI,gBAAe,GAAW,CAC5B,OAAW,KAAS,GAClB,GAAO,KAAK,OAGf,KACC,EAAU,GAAU,EAAM,KAAK,EAAU,IACtC,KACC,GAAS,IAAM,EAAO,gBAG1B,GAAY,IAcT,YAAwB,EAA8B,CAC3D,MAAO,CACL,MAAQ,EAAG,YACX,OAAQ,EAAG,cAWR,YAA+B,EAA8B,CAClE,MAAO,CACL,MAAQ,EAAG,YACX,OAAQ,EAAG,cAyBR,YACL,EACyB,CACzB,MAAO,IACJ,KACC,EAAI,GAAY,EAAS,QAAQ,IACjC,EAAU,GAAY,GACnB,KACC,EAAO,CAAC,CAAE,YAAa,IAAW,GAClC,GAAS,IAAM,EAAS,UAAU,IAClC,EAAI,IAAM,GAAe,MAG7B,EAAU,GAAe,KC9FxB,YAA0B,EAAgC,CAC/D,MAAO,CACL,EAAG,EAAG,WACN,EAAG,EAAG,WAaH,YACL,EAC2B,CAC3B,MAAO,GACL,EAAU,EAAI,UACd,EAAU,OAAQ,WAEjB,KACC,EAAI,IAAM,GAAiB,IAC3B,EAAU,GAAiB,KAe1B,YACL,EAAiB,EAAY,GACR,CACrB,MAAO,IAAmB,GACvB,KACC,EAAI,CAAC,CAAE,OAAQ,CACb,GAAM,GAAU,GAAe,GACzB,EAAU,GAAsB,GACtC,MAAO,IACL,EAAQ,OAAS,EAAQ,OAAS,IAGtC,KC9EC,YACL,EACM,CACN,GAAI,YAAc,kBAChB,EAAG,aAEH,MAAM,IAAI,OAAM,mBCQpB,GAAM,IAA4C,CAChD,OAAQ,GAAkB,2BAC1B,OAAQ,GAAkB,4BAcrB,YAAmB,EAAuB,CAC/C,MAAO,IAAQ,GAAM,QAchB,YAAmB,EAAc,EAAsB,CAC5D,AAAI,GAAQ,GAAM,UAAY,GAC5B,GAAQ,GAAM,QAYX,YAAqB,EAAmC,CAC7D,GAAM,GAAK,GAAQ,GACnB,MAAO,GAAU,EAAI,UAClB,KACC,EAAI,IAAM,EAAG,SACb,EAAU,EAAG,UClCnB,YAAiC,EAA0B,CACzD,OAAQ,EAAG,aAGJ,YACA,aACA,WACH,MAAO,WAIP,MAAO,GAAG,mBAaT,aAA+C,CACpD,MAAO,GAAyB,OAAQ,WACrC,KACC,EAAO,GAAM,CAAE,GAAG,SAAW,EAAG,UAChC,EAAI,GAAO,EACT,KAAM,GAAU,UAAY,SAAW,SACvC,KAAM,EAAG,IACT,OAAQ,CACN,EAAG,iBACH,EAAG,sBAGP,EAAO,CAAC,CAAE,UAAW,CACnB,GAAI,IAAS,SAAU,CACrB,GAAM,GAAS,KACf,GAAI,MAAO,IAAW,YACpB,MAAO,CAAC,GAAwB,GAEpC,MAAO,KAET,MCnEC,aAA4B,CACjC,MAAO,IAAI,KAAI,SAAS,MAQnB,YAAqB,EAAgB,CAC1C,SAAS,KAAO,EAAI,KAUf,aAAuC,CAC5C,MAAO,IAAI,GCvBN,aAAmC,CACxC,MAAO,UAAS,KAAK,UAAU,GAa1B,YAAyB,EAAoB,CAClD,GAAM,GAAK,GAAc,KACzB,EAAG,KAAO,EACV,EAAG,iBAAiB,QAAS,GAAM,EAAG,mBACtC,EAAG,QAUE,aAAiD,CACtD,MAAO,GAA2B,OAAQ,cACvC,KACC,EAAI,IACJ,EAAU,MACV,EAAO,GAAQ,EAAK,OAAS,GAC7B,MASC,aAAwD,CAC7D,MAAO,MACJ,KACC,EAAU,GAAM,EAAG,GAAW,QAAQ,UChCrC,YAAoB,EAAoC,CAC7D,GAAM,GAAQ,WAAW,GACzB,MAAO,IAA0B,GAC/B,EAAM,YAAY,IAAM,EAAK,EAAM,WAElC,KACC,EAAU,EAAM,UASf,aAAwC,CAC7C,MAAO,GAAU,OAAQ,eACtB,KACC,GAAM,SAgBL,YACL,EAA6B,EACd,CACf,MAAO,GACJ,KACC,EAAU,GAAU,EAAS,IAAY,IC/CxC,YACL,EAAmB,EAAuB,CAAE,YAAa,eACnC,CACtB,MAAO,IAAK,MAAM,GAAG,IAAO,IACzB,KACC,EAAO,GAAO,EAAI,SAAW,MAc5B,YACL,EAAmB,EACJ,CACf,MAAO,IAAQ,EAAK,GACjB,KACC,EAAU,GAAO,EAAI,QACrB,GAAY,IAYX,YACL,EAAmB,EACG,CACtB,GAAM,GAAM,GAAI,WAChB,MAAO,IAAQ,EAAK,GACjB,KACC,EAAU,GAAO,EAAI,QACrB,EAAI,GAAO,EAAI,gBAAgB,EAAK,aACpC,GAAY,ICtCX,aAA6C,CAClD,MAAO,CACL,EAAG,KAAK,IAAI,EAAG,aACf,EAAG,KAAK,IAAI,EAAG,cASZ,YACL,CAAE,IAAG,KACC,CACN,OAAO,SAAS,GAAK,EAAG,GAAK,GAUxB,aAA2D,CAChE,MAAO,GACL,EAAU,OAAQ,SAAU,CAAE,QAAS,KACvC,EAAU,OAAQ,SAAU,CAAE,QAAS,MAEtC,KACC,EAAI,IACJ,EAAU,OCnCT,aAAyC,CAC9C,MAAO,CACL,MAAQ,WACR,OAAQ,aAWL,aAAuD,CAC5D,MAAO,GAAU,OAAQ,SAAU,CAAE,QAAS,KAC3C,KACC,EAAI,IACJ,EAAU,OCST,aAA+C,CACpD,MAAO,GAAc,CACnB,KACA,OAEC,KACC,EAAI,CAAC,CAAC,EAAQ,KAAW,EAAE,SAAQ,UACnC,GAAY,IAYX,YACL,EAAiB,CAAE,YAAW,WACR,CACtB,GAAM,GAAQ,EACX,KACC,EAAwB,SAItB,EAAU,EAAc,CAAC,EAAO,IACnC,KACC,EAAI,IAAuB,EACzB,EAAG,EAAG,WACN,EAAG,EAAG,cAKZ,MAAO,GAAc,CAAC,EAAS,EAAW,IACvC,KACC,EAAI,CAAC,CAAC,CAAE,UAAU,CAAE,SAAQ,QAAQ,CAAE,IAAG,QAAU,EACjD,OAAQ,CACN,EAAG,EAAO,EAAI,EACd,EAAG,EAAO,EAAI,EAAI,GAEpB,WChCD,YACL,EAAgB,CAAE,OACH,CAGf,GAAM,GAAM,EAAwB,EAAQ,WACzC,KACC,EAAI,CAAC,CAAE,UAAW,IAItB,MAAO,GACJ,KACC,GAAS,IAAM,EAAK,CAAE,QAAS,GAAM,SAAU,KAC/C,EAAI,GAAW,EAAO,YAAY,IAClC,GAAY,GACZ,MCLN,GAAM,IAAS,GAAkB,aAC3B,GAAiB,KAAK,MAAM,GAAO,aACzC,GAAO,KAAO,GAAG,GAAI,KAAI,GAAO,KAAM,QAW/B,aAAiC,CACtC,MAAO,IAUF,YAAiB,EAAqB,CAC3C,MAAO,IAAO,SAAS,SAAS,GAW3B,WACL,EAAkB,EACV,CACR,MAAO,OAAO,IAAU,YACpB,GAAO,aAAa,GAAK,QAAQ,IAAK,EAAM,YAC5C,GAAO,aAAa,GC1BnB,YACL,EAAS,EAAmB,SACP,CACrB,MAAO,IAAkB,sBAAsB,KAAS,GAanD,YACL,EAAS,EAAmB,SACL,CACvB,MAAO,GAAY,sBAAsB,KAAS,GC5GpD,OAAwB,SCUjB,YACL,EAAiB,EAAQ,EACnB,CACN,EAAG,aAAa,WAAY,EAAM,YAQ7B,YACL,EACM,CACN,EAAG,gBAAgB,YASd,YACL,EAAiB,EACX,CACN,EAAG,aAAa,gBAAiB,QACjC,EAAG,MAAM,IAAM,IAAI,MAQd,YACL,EACM,CACN,GAAM,GAAQ,GAAK,SAAS,EAAG,MAAM,IAAK,IAC1C,EAAG,gBAAgB,iBACnB,EAAG,MAAM,IAAM,GACX,GACF,OAAO,SAAS,EAAG,GC1ChB,YACL,EAAiB,EACX,CACN,EAAG,aAAa,gBAAiB,GAQ5B,YACL,EACM,CACN,EAAG,gBAAgB,iBAWd,YACL,EAAiB,EACX,CACN,EAAG,UAAU,OAAO,uBAAwB,GAQvC,YACL,EACM,CACN,EAAG,UAAU,OAAO,wBCvCf,YACL,EAAiB,EACX,CACN,EAAG,kBAAmB,UAAY,EAW7B,YACL,EAAiB,EACX,CACN,EAAG,aAAa,gBAAiB,GAQ5B,YACL,EACM,CACN,EAAG,gBAAgB,iBC5Bd,YACL,EAAiB,EACX,CACN,EAAG,aAAa,gBAAiB,GAQ5B,YACL,EACM,CACN,EAAG,gBAAgB,iBCdd,YACL,EAAiB,EACX,CACN,EAAG,aAAa,gBAAiB,GAQ5B,YACL,EACM,CACN,EAAG,gBAAgB,iBCZd,YACL,EAAsB,EAChB,CACN,EAAG,YAAc,EAQZ,YACL,EACM,CACN,EAAG,YAAc,EAAY,sBCO/B,YAAqB,EAAiB,EAA8B,CAGlE,GAAI,MAAO,IAAU,UAAY,MAAO,IAAU,SAChD,EAAG,WAAa,EAAM,mBAGb,YAAiB,MAC1B,EAAG,YAAY,WAGN,MAAM,QAAQ,GACvB,OAAW,KAAQ,GACjB,GAAY,EAAI,GAiBf,WACL,EAAa,KAAkC,EAClC,CACb,GAAM,GAAK,SAAS,cAAc,GAGlC,GAAI,EACF,OAAW,KAAQ,QAAO,KAAK,GAC7B,AAAI,MAAO,GAAW,IAAU,UAC9B,EAAG,aAAa,EAAM,EAAW,IAC1B,EAAW,IAClB,EAAG,aAAa,EAAM,IAG5B,OAAW,KAAS,GAClB,GAAY,EAAI,GAGlB,MAAO,GChEF,YAAkB,EAAe,EAAmB,CACzD,GAAI,GAAI,EACR,GAAI,EAAM,OAAS,EAAG,CACpB,KAAO,EAAM,KAAO,KAAO,EAAE,EAAI,GAAG,CACpC,MAAO,GAAG,EAAM,UAAU,EAAG,QAE/B,MAAO,GAmBF,YAAe,EAAuB,CAC3C,GAAI,EAAQ,IAAK,CACf,GAAM,GAAS,CAAG,IAAQ,KAAO,IAAO,IACxC,MAAO,GAAK,IAAQ,MAAY,KAAM,QAAQ,UAE9C,OAAO,GAAM,WClCV,YACL,EAAiB,EACX,CACN,OAAQ,OAGD,GACH,EAAG,YAAc,EAAY,sBAC7B,UAGG,GACH,EAAG,YAAc,EAAY,qBAC7B,cAIA,EAAG,YAAc,EAAY,sBAAuB,GAAM,KASzD,YACL,EACM,CACN,EAAG,YAAc,EAAY,6BAWxB,YACL,EAAiB,EACX,CACN,EAAG,YAAY,GAQV,YACL,EACM,CACN,EAAG,UAAY,GCzDV,YACL,EAAiB,EACX,CACN,EAAG,MAAM,IAAM,GAAG,MAQb,YACL,EACM,CACN,EAAG,MAAM,IAAM,GAwBV,YACL,EAAiB,EACX,CACN,GAAM,GAAa,EAAG,kBACtB,EAAW,MAAM,OAAS,GAAG,EAAQ,EAAI,EAAW,cAQ/C,YACL,EACM,CACN,GAAM,GAAa,EAAG,kBACtB,EAAW,MAAM,OAAS,GCtDrB,YACL,EAAiB,EACX,CACN,EAAG,iBAAkB,YAAY,GAS5B,YACL,EAAiB,EACX,CACN,EAAG,iBAAkB,aAAa,gBAAiB,GCf9C,YACL,EAAiB,EACX,CACN,EAAG,aAAa,gBAAiB,GAQ5B,YACL,EACM,CACN,EAAG,gBAAgB,iBCdd,YACL,EAAiB,EACX,CACN,EAAG,aAAa,gBAAiB,GAQ5B,YACL,EACM,CACN,EAAG,gBAAgB,iBAWd,YACL,EAAiB,EACX,CACN,EAAG,MAAM,IAAM,GAAG,MAQb,YACL,EACM,CACN,EAAG,MAAM,IAAM,GCnCV,YAA+B,EAAyB,CAC7D,MACE,GAAC,SAAD,CACE,MAAM,uBACN,MAAO,EAAY,kBACnB,wBAAuB,IAAI,aCJjC,GAAW,IAAX,UAAW,EAAX,CACE,WAAS,GAAT,SACA,WAAS,GAAT,WAFS,aAiBX,YACE,EAA2C,EAC9B,CACb,GAAM,GAAS,EAAO,EAChB,EAAS,EAAO,EAGhB,EAAU,OAAO,KAAK,EAAS,OAClC,OAAO,GAAO,CAAC,EAAS,MAAM,IAC9B,IAAI,GAAO,CAAC,EAAC,MAAD,KAAM,GAAY,MAC9B,OACA,MAAM,EAAG,IAGN,EAAM,GAAI,KAAI,EAAS,UAC7B,MAAI,IAAQ,qBACV,EAAI,aAAa,IAAI,IAAK,OAAO,QAAQ,EAAS,OAC/C,OAAO,CAAC,CAAC,CAAE,KAAW,GACtB,OAAO,CAAC,EAAW,CAAC,KAAW,GAAG,KAAa,IAAQ,OAAQ,KAKlE,EAAC,IAAD,CAAG,KAAM,GAAG,IAAO,MAAM,yBAAyB,SAAU,IAC1D,EAAC,UAAD,CACE,MAAO,CAAC,4BAA6B,GAAG,EACpC,CAAC,uCACD,IACF,KAAK,KACP,gBAAe,EAAS,MAAM,QAAQ,IAErC,EAAS,GAAK,EAAC,MAAD,CAAK,MAAM,mCAC1B,EAAC,KAAD,CAAI,MAAM,2BAA2B,EAAS,OAC7C,EAAS,GAAK,EAAS,KAAK,OAAS,GACpC,EAAC,IAAD,CAAG,MAAM,4BACN,GAAS,EAAS,KAAM,MAG5B,EAAS,GAAK,EAAQ,OAAS,GAC9B,EAAC,IAAD,CAAG,MAAM,2BACN,EAAY,8BAA8B,KAAM,KAmBtD,YACL,EACa,CACb,GAAM,GAAY,EAAO,GAAG,MACtB,EAAO,CAAC,GAAG,GAGX,EAAS,EAAK,UAAU,GAAO,CAAC,EAAI,SAAS,SAAS,MACtD,CAAC,GAAW,EAAK,OAAO,EAAQ,GAGlC,EAAQ,EAAK,UAAU,GAAO,EAAI,MAAQ,GAC9C,AAAI,IAAU,IACZ,GAAQ,EAAK,QAGf,GAAM,GAAO,EAAK,MAAM,EAAG,GACrB,EAAO,EAAK,MAAM,GAGlB,EAAW,CACf,GAAqB,EAAS,EAAc,CAAE,EAAC,GAAU,IAAU,IACnE,GAAG,EAAK,IAAI,GAAW,GAAqB,EAAS,IACrD,GAAG,EAAK,OAAS,CACf,EAAC,UAAD,CAAS,MAAM,0BACb,EAAC,UAAD,CAAS,SAAU,IAChB,EAAK,OAAS,GAAK,EAAK,SAAW,EAChC,EAAY,0BACZ,EAAY,2BAA4B,EAAK,SAG/C,EAAK,IAAI,GAAW,GAAqB,EAAS,MAEtD,IAIN,MACE,GAAC,KAAD,CAAI,MAAM,0BACP,GCpHA,YAA2B,EAAiC,CACjE,MACE,GAAC,KAAD,CAAI,MAAM,oBACP,OAAO,QAAQ,GAAO,IAAI,CAAC,CAAC,EAAK,KAChC,EAAC,KAAD,CAAI,MAAO,oCAAoC,KAC5C,MAAO,IAAU,SAAW,GAAM,GAAS,KCN/C,YAAqB,EAAiC,CAC3D,MACE,GAAC,MAAD,CAAK,MAAM,0BACT,EAAC,MAAD,CAAK,MAAM,qBACR,ICUT,YAAuB,EAA+B,CACpD,GAAM,GAAS,KAGT,EAAM,GAAI,KAAI,MAAM,EAAQ,WAAY,EAAO,MACrD,MACE,GAAC,KAAD,CAAI,MAAM,oBACR,EAAC,IAAD,CAAG,KAAM,EAAI,WAAY,MAAM,oBAC5B,EAAQ,QAiBV,YAA+B,EAAkC,CACtE,GAAM,GAAS,KAGT,CAAC,CAAE,GAAW,EAAO,KAAK,MAAM,eAChC,EACJ,EAAS,KAAK,CAAC,CAAE,UAAS,aACxB,IAAY,GAAW,EAAQ,SAAS,KACpC,EAAS,GAGjB,MACE,GAAC,MAAD,CAAK,MAAM,cACT,EAAC,SAAD,CACE,MAAM,sBACN,aAAY,EAAY,yBAEvB,EAAO,OAEV,EAAC,KAAD,CAAI,MAAM,oBACP,EAAS,IAAI,MlBPtB,GAAI,IAAQ,EAiBL,YACL,EAAiB,CAAE,aACI,CACvB,GAAM,GAAa,EAAG,GACnB,KACC,EAAU,GAAS,CACjB,GAAM,GAAY,EAAM,QAAQ,eAChC,MAAI,aAAqB,aAChB,EACL,GAAG,EAAY,QAAS,GACrB,IAAI,GAAS,EAAU,EAAO,YAG9B,KAKb,MAAO,GACL,EAAU,KAAK,EAAwB,SACvC,GAEC,KACC,EAAI,IAAM,CACR,GAAM,GAAU,GAAe,GAE/B,MAAO,CACL,OAAQ,AAFM,GAAsB,GAEpB,MAAQ,EAAQ,SAGpC,EAAwB,WAevB,YACL,EAAiB,EACiB,CAClC,GAAM,GAAY,GAAI,GAatB,GAZA,EACG,KACC,GAAe,GAAW,aAEzB,UAAU,CAAC,CAAC,CAAE,UAAU,KAAW,CAClC,AAAI,GAAU,EACZ,GAAa,GAEb,GAAe,KAInB,WAAY,cAAe,CAC7B,GAAM,GAAS,EAAG,QAAQ,OAC1B,EAAO,GAAK,UAAU,OACtB,EAAO,aACL,GAAsB,EAAO,IAC7B,GAKJ,MAAO,IAAe,EAAI,GACvB,KACC,EAAI,GACJ,EAAI,GAAU,GAAE,IAAK,GAAO,KmBxG3B,YACL,EAAwB,CAAE,UAAS,UACd,CACrB,MAAO,GACJ,KACC,EAAI,GAAU,EAAO,QAAQ,wBAC7B,EAAO,GAAW,IAAO,GACzB,GAAU,GACV,GAAM,IAeL,YACL,EAAwB,EACQ,CAChC,GAAM,GAAY,GAAI,GACtB,SAAU,UAAU,IAAM,CACxB,EAAG,aAAa,OAAQ,IACxB,EAAG,mBAIE,GAAa,EAAI,GACrB,KACC,EAAI,GACJ,GAAM,CAAE,IAAK,KCjEnB,GAAM,IAAW,GAAc,SAgBxB,YACL,EACkC,CAClC,UAAe,EAAI,IACnB,GAAe,GAAU,GAAY,IAG9B,EAAG,CAAE,IAAK,ICGZ,YACL,EAAiB,CAAE,UAAS,YAAW,UACP,CAChC,MAAO,GAGL,GAAG,EAAY,aAAc,GAC1B,IAAI,GAAS,GAAe,EAAO,CAAE,eAGxC,GAAG,EAAY,qBAAsB,GAClC,IAAI,GAAS,GAAe,IAG/B,GAAG,EAAY,UAAW,GACvB,IAAI,GAAS,GAAa,EAAO,CAAE,UAAS,aCC5C,YACL,EAAkB,CAAE,UACA,CACpB,MAAO,GACJ,KACC,EAAU,GAAW,EACnB,EAAG,IACH,EAAG,IAAO,KAAK,GAAM,OAEpB,KACC,EAAI,GAAS,EAAE,UAAS,aAiB3B,YACL,EAAiB,EACc,CAC/B,GAAM,GAAY,GAAI,GACtB,SACG,KACC,EAAU,IAET,UAAU,CAAC,CAAE,UAAS,UAAW,CAChC,GAAiB,EAAI,GACrB,AAAI,EACF,GAAe,EAAI,QAEnB,GAAiB,KAIlB,GAAY,EAAI,GACpB,KACC,EAAI,GACJ,EAAI,GAAU,GAAE,IAAK,GAAO,KCjClC,YAAkB,CAAE,aAAgD,CAClE,GAAI,CAAC,GAAQ,mBACX,MAAO,GAAG,IAGZ,GAAM,GAAa,EAChB,KACC,EAAI,CAAC,CAAE,OAAQ,CAAE,QAAU,GAC3B,GAAY,EAAG,GACf,EAAI,CAAC,CAAC,EAAG,KAAO,CAAC,EAAI,EAAG,IACxB,EAAwB,IAItB,EAAU,EAAc,CAAC,EAAW,IACvC,KACC,EAAO,CAAC,CAAC,CAAE,UAAU,CAAC,CAAE,MAAQ,KAAK,IAAI,EAAI,EAAO,GAAK,KACzD,EAAI,CAAC,CAAC,CAAE,CAAC,MAAgB,GACzB,KAIE,EAAU,GAAY,UAC5B,MAAO,GAAc,CAAC,EAAW,IAC9B,KACC,EAAI,CAAC,CAAC,CAAE,UAAU,KAAY,EAAO,EAAI,KAAO,CAAC,GACjD,IACA,EAAU,GAAU,EAAS,EAAU,EAAG,KAC1C,EAAU,KAgBT,YACL,EAAiB,EACG,CACpB,MAAO,IAAM,IAAM,CACjB,GAAM,GAAS,iBAAiB,GAChC,MAAO,GACL,EAAO,WAAa,UACpB,EAAO,WAAa,oBAGrB,KACC,GAAkB,GAAiB,GAAK,GAAS,IACjD,EAAI,CAAC,CAAC,EAAQ,CAAE,UAAU,KAAa,EACrC,OAAQ,EAAS,EAAS,EAC1B,SACA,YAEF,EAAqB,CAAC,EAAG,IACvB,EAAE,SAAW,EAAE,QACf,EAAE,SAAW,EAAE,QACf,EAAE,SAAW,EAAE,QAEjB,GAAY,IAeX,YACL,EAAiB,CAAE,UAAS,SACG,CAC/B,GAAM,GAAY,GAAI,GACtB,SACG,KACC,EAAwB,UACxB,GAAkB,GAClB,EAAU,IAET,UAAU,CAAC,CAAC,CAAE,UAAU,CAAE,aAAc,CACvC,AAAI,EACF,GAAe,EAAI,EAAS,SAAW,UAEvC,GAAiB,KAIzB,EAAM,UAAU,GAAQ,EAAU,KAAK,IAChC,EACJ,KACC,EAAI,GAAU,GAAE,IAAK,GAAO,KC/G3B,YACL,EAAwB,CAAE,YAAW,WACZ,CACzB,MAAO,IAAgB,EAAI,CAAE,UAAS,cACnC,KACC,EAAI,CAAC,CAAE,OAAQ,CAAE,QAAU,CACzB,GAAM,CAAE,UAAW,GAAe,GAClC,MAAO,CACL,OAAQ,GAAK,KAGjB,EAAwB,WAevB,YACL,EAAiB,EACmB,CACpC,GAAM,GAAY,GAAI,GACtB,EACG,KACC,EAAU,IAET,UAAU,CAAC,CAAE,YAAa,CACzB,AAAI,EACF,GAAoB,EAAI,UAExB,GAAsB,KAI9B,GAAM,GAAW,GAA+B,cAChD,MAAI,OAAO,IAAa,YACf,EAGF,GAAiB,EAAU,GAC/B,KACC,EAAI,GACJ,EAAI,GAAU,GAAE,IAAK,GAAO,KChE3B,YACL,EAAiB,CAAE,YAAW,WACZ,CAGlB,GAAM,GAAU,EACb,KACC,EAAI,CAAC,CAAE,YAAa,GACpB,KAIE,EAAU,EACb,KACC,EAAU,IAAM,GAAiB,GAC9B,KACC,EAAI,CAAC,CAAE,YAAc,EACnB,IAAQ,EAAG,UACX,OAAQ,EAAG,UAAY,KAEzB,EAAwB,aAMhC,MAAO,GAAc,CAAC,EAAS,EAAS,IACrC,KACC,EAAI,CAAC,CAAC,EAAQ,CAAE,MAAK,UAAU,CAAE,OAAQ,CAAE,KAAK,KAAM,CAAE,cACtD,GAAS,KAAK,IAAI,EAAG,EACjB,KAAK,IAAI,EAAG,EAAS,EAAI,GACzB,KAAK,IAAI,EAAG,EAAS,EAAI,IAEtB,CACL,OAAQ,EAAM,EACd,SACA,OAAQ,EAAM,GAAU,KAG5B,EAAqB,CAAC,EAAG,IACvB,EAAE,SAAW,EAAE,QACf,EAAE,SAAW,EAAE,QACf,EAAE,SAAW,EAAE,SC/ChB,YACL,EACqB,CACrB,GAAM,GAAO,aAAa,QAAQ,SAAS,cACrC,EAAU,KAAK,MAAM,IAAS,CAClC,MAAO,EAAO,UAAU,GACtB,WAAW,EAAM,aAAa,wBAAyB,UAKrD,EAAW,EAAG,GAAG,GACpB,KACC,GAAS,GAAS,EAAU,EAAO,UAChC,KACC,GAAM,KAGV,EAAU,EAAO,KAAK,IAAI,EAAG,EAAQ,SACrC,EAAI,GAAU,EACZ,MAAO,EAAO,QAAQ,GACtB,MAAO,CACL,OAAS,EAAM,aAAa,wBAC5B,QAAS,EAAM,aAAa,yBAC5B,OAAS,EAAM,aAAa,4BAGhC,GAAY,IAIhB,SAAS,UAAU,GAAW,CAC5B,aAAa,QAAQ,SAAS,aAAc,KAAK,UAAU,MAItD,EAUF,YACL,EACgC,CAChC,GAAM,GAAY,GAAI,GAGtB,EAAU,UAAU,GAAW,CAC7B,OAAW,CAAC,EAAK,IAAU,QAAO,QAAQ,EAAQ,OAChD,AAAI,MAAO,IAAU,UACnB,SAAS,KAAK,aAAa,iBAAiB,IAAO,GAGvD,OAAS,GAAQ,EAAG,EAAQ,EAAO,OAAQ,IAAS,CAClD,GAAM,GAAQ,EAAO,GAAO,mBAC5B,AAAI,YAAiB,cACnB,GAAM,OAAS,EAAQ,QAAU,MAKvC,GAAM,GAAS,EAA8B,QAAS,GACtD,MAAO,IAAa,GACjB,KACC,EAAI,GACJ,EAAI,GAAU,GAAE,IAAK,GAAO,KCzHlC,OAAwB,SAyBjB,YACL,CAAE,UACI,CACN,AAAI,WAAY,eACd,GAAI,GAA8B,GAAc,CAC9C,GAAI,YAAY,kDACb,GAAG,UAAW,GAAM,EAAW,KAAK,MAEtC,UAAU,IAAM,EAAO,KAAK,EAAY,sBC+C/C,YAAoB,EAA0B,CAC5C,GAAI,EAAK,OAAS,EAChB,MAAO,GAGT,GAAM,CAAC,EAAM,GAAQ,EAClB,KAAK,CAAC,EAAG,IAAM,EAAE,OAAS,EAAE,QAC5B,IAAI,GAAO,EAAI,QAAQ,SAAU,KAGhC,EAAQ,EACZ,GAAI,IAAS,EACX,EAAQ,EAAK,WAEb,MAAO,EAAK,WAAW,KAAW,EAAK,WAAW,IAChD,IAGJ,GAAM,GAAS,KACf,MAAO,GAAK,IAAI,GACd,EAAI,QAAQ,EAAK,MAAM,EAAG,GAAQ,EAAO,OA6BtC,YACL,CAAE,YAAW,YAAW,aAClB,CACN,GAAM,GAAS,KACf,GAAI,SAAS,WAAa,QACxB,OAGF,AAAI,qBAAuB,UACzB,SAAQ,kBAAoB,SAG5B,EAAU,OAAQ,gBACf,UAAU,IAAM,CACf,QAAQ,kBAAoB,UAKlC,GAAM,GAAU,GAA4B,kBAC5C,AAAI,MAAO,IAAY,aACrB,GAAQ,KAAO,EAAQ,MAGzB,GAAM,GAAQ,GAAW,GAAI,KAAI,cAAe,EAAO,OACpD,KACC,EAAI,GAAW,GAAW,EAAY,MAAO,GAC1C,IAAI,GAAQ,EAAK,eAEpB,EAAU,GAAQ,EAAsB,SAAS,KAAM,SACpD,KACC,EAAO,GAAM,CAAC,EAAG,SAAW,CAAC,EAAG,SAChC,EAAU,GAAM,CAGd,GAAI,EAAG,iBAAkB,SAAS,CAChC,GAAM,GAAK,EAAG,OAAO,QAAQ,KAC7B,GAAI,GAAM,CAAC,EAAG,OAAQ,CACpB,GAAM,GAAM,GAAI,KAAI,EAAG,MAOvB,GAJA,EAAI,OAAS,GACb,EAAI,KAAO,GAIT,EAAI,WAAa,SAAS,UAC1B,EAAK,SAAS,EAAI,YAElB,SAAG,iBACI,EAAG,CACR,IAAK,GAAI,KAAI,EAAG,SAKxB,MAAO,OAIb,MAIE,EAAO,EAAyB,OAAQ,YAC3C,KACC,EAAO,GAAM,EAAG,QAAU,MAC1B,EAAI,GAAO,EACT,IAAK,GAAI,KAAI,SAAS,MACtB,OAAQ,EAAG,SAEb,MAIJ,EAAM,EAAO,GACV,KACC,EAAqB,CAAC,EAAG,IAAM,EAAE,IAAI,OAAS,EAAE,IAAI,MACpD,EAAI,CAAC,CAAE,SAAU,IAEhB,UAAU,GAGf,GAAM,GAAY,EACf,KACC,EAAwB,YACxB,EAAU,GAAO,GAAQ,EAAI,MAC1B,KACC,GAAW,IACT,IAAY,GACL,MAIb,MAIJ,EACG,KACC,GAAO,IAEN,UAAU,CAAC,CAAE,SAAU,CACtB,QAAQ,UAAU,GAAI,GAAI,GAAG,OAInC,GAAM,GAAM,GAAI,WAChB,EACG,KACC,EAAU,GAAO,EAAI,QACrB,EAAI,GAAO,EAAI,gBAAgB,EAAK,eAEnC,UAAU,GAGf,EACG,KACC,GAAK,IAEJ,UAAU,GAAe,CACxB,OAAW,KAAY,CAGrB,QACA,sBACA,oBACA,yBAGA,+BACA,gCACA,mCACA,qCACA,4BACC,CACD,GAAM,GAAS,GAAW,GACpB,EAAS,GAAW,EAAU,GACpC,AACE,MAAO,IAAW,aAClB,MAAO,IAAW,aAElB,GAAe,EAAQ,MAMjC,EACG,KACC,GAAK,GACL,EAAI,IAAM,GAAoB,cAC9B,EAAU,GAAM,EAAG,GAAG,EAAY,SAAU,KAC5C,GAAU,GAAM,CACd,GAAM,GAAS,GAAc,UAC7B,GAAI,EAAG,IAAK,CACV,OAAW,KAAQ,GAAG,oBACpB,EAAO,aAAa,EAAM,EAAG,aAAa,IAC5C,UAAe,EAAI,GAGZ,GAAI,GAAW,GAAY,CAChC,EAAO,OAAS,IAAM,EAAS,iBAKjC,UAAO,YAAc,EAAG,YACxB,GAAe,EAAI,GACZ,MAIV,YAGL,EAAM,EAAO,GACV,KACC,GAAO,IAEN,UAAU,CAAC,CAAE,MAAK,YAAa,CAC9B,AAAI,EAAI,MAAQ,CAAC,EACf,GAAgB,EAAI,MAEpB,GAAkB,GAAU,CAAE,EAAG,MAKzC,EACG,KACC,GAAU,GACV,GAAa,KACb,EAAwB,WAEvB,UAAU,CAAC,CAAE,YAAa,CACzB,QAAQ,aAAa,EAAQ,MAInC,EAAM,EAAO,GACV,KACC,GAAY,EAAG,GACf,EAAO,CAAC,CAAC,EAAG,KAAO,EAAE,IAAI,WAAa,EAAE,IAAI,UAC5C,EAAI,CAAC,CAAC,CAAE,KAAW,IAElB,UAAU,CAAC,CAAE,YAAa,CACzB,GAAkB,GAAU,CAAE,EAAG,MChVzC,OAAuB,SCAvB,OAAuB,SAqChB,YACL,EAC0B,CAC1B,GAAM,GAAY,GAAI,QAAO,EAAO,UAAW,OACzC,EAAY,CAAC,EAAY,EAAc,IACpC,GAAG,4BAA+B,WAI3C,MAAO,AAAC,IAAkB,CACxB,EAAQ,EACL,QAAQ,gBAAiB,KACzB,OAGH,GAAM,GAAQ,GAAI,QAAO,MAAM,EAAO,cACpC,EACG,QAAQ,uBAAwB,QAChC,QAAQ,EAAW,QACnB,OAGL,MAAO,IAAS,eAAW,GACxB,QAAQ,EAAO,GACf,QAAQ,8BAA+B,OCvBvC,YAA0B,EAAuB,CACtD,MAAO,GACJ,MAAM,cACJ,IAAI,CAAC,EAAO,IAAU,EAAQ,EAC3B,EAAM,QAAQ,+BAAgC,MAC9C,GAEH,KAAK,IACP,QAAQ,kCAAmC,IAC3C,OCtCE,GAAW,IAAX,UAAW,EAAX,CACL,qBACA,qBACA,qBACA,yBAJgB,aA2EX,YACL,EAC+B,CAC/B,MAAO,GAAQ,OAAS,EAUnB,YACL,EAC+B,CAC/B,MAAO,GAAQ,OAAS,EAUnB,YACL,EACgC,CAChC,MAAO,GAAQ,OAAS,EC3E1B,YACE,CAAE,SAAQ,OAAM,SACH,CAGb,AAAI,EAAO,KAAK,SAAW,GAAK,EAAO,KAAK,KAAO,MACjD,GAAO,KAAO,CACZ,EAAY,wBAIZ,EAAO,YAAc,aACvB,GAAO,UAAY,EAAY,4BAQjC,GAAM,GAAyB,CAC7B,SANe,EAAY,0BAC1B,MAAM,WACN,OAAO,SAKR,YAAa,GAAQ,mBAIvB,MAAO,CAAE,SAAQ,OAAM,QAAO,WAmBzB,YACL,EAAa,EACC,CACd,GAAM,GAAS,KACT,EAAS,GAAI,QAAO,GAGpB,EAAM,GAAI,GACV,EAAM,GAAY,EAAQ,CAAE,QAC/B,KACC,EAAI,GAAW,CACb,GAAI,GAAsB,GACxB,OAAW,KAAU,GAAQ,KAAK,MAChC,OAAW,KAAY,GACrB,EAAS,SAAW,GAAG,GAAI,KAAI,EAAS,SAAU,EAAO,QAE/D,MAAO,KAET,MAIJ,UAAK,GACF,KACC,EAAqC,GAAS,EAC5C,KAAM,GAAkB,MACxB,KAAM,GAAiB,OAGxB,UAAU,EAAI,KAAK,KAAK,IAGtB,CAAE,MAAK,OCxGT,aAAsC,CAC3C,GAAM,GAAS,KACf,GAAuB,GAAI,KAAI,mBAAoB,EAAO,OACvD,UAAU,GAAY,CAErB,AADc,GAAkB,qBAC1B,YAAY,GAAsB,MCkDvC,YACL,EAAsB,CAAE,OACC,CACzB,GAAM,GAAK,gCAAU,YAAa,GAG5B,EAAS,GAAkB,GAC3B,EAAS,EACb,EAAU,EAAI,SACd,EAAU,EAAI,SAAS,KAAK,GAAM,KAEjC,KACC,EAAI,IAAM,EAAG,EAAG,QAChB,KAIE,EAAW,KACjB,MAAI,GAAS,aAAa,IAAI,MAC5B,IAAU,SAAU,IACpB,EACG,KACC,EAAO,IACP,GAAK,IAEJ,UAAU,IAAM,CACf,EAAG,MAAQ,EAAS,aAAa,IAAI,KACrC,GAAgB,MAKjB,EAAc,CAAC,EAAQ,IAC3B,KACC,EAAI,CAAC,CAAC,EAAO,KAAY,EAAE,QAAO,YAYjC,YACL,EAAsB,CAAE,MAAK,OACyB,CACtD,GAAM,GAAY,GAAI,GAGtB,SACG,KACC,EAAwB,SACxB,EAAI,CAAC,CAAE,WAAiC,EACtC,KAAM,GAAkB,MACxB,KAAM,MAGP,UAAU,EAAI,KAAK,KAAK,IAG7B,EACG,KACC,EAAwB,UAEvB,UAAU,CAAC,CAAE,WAAY,CACxB,AAAI,EACF,IAAU,SAAU,GACpB,GAA0B,EAAI,KAE9B,GAA4B,KAKpC,EAAU,EAAG,KAAO,SACjB,KACC,GAAU,EAAU,KAAK,GAAS,MAEjC,UAAU,IAAM,GAAgB,IAG9B,GAAiB,EAAI,CAAE,MAAK,QAChC,KACC,EAAI,GACJ,EAAI,GAAU,GAAE,IAAK,GAAO,KCtF3B,YACL,EAAiB,CAAE,OAAqB,CAAE,UACL,CACrC,GAAM,GAAY,GAAI,GAChB,EAAY,GAAsB,EAAG,eACxC,KACC,EAAO,UAIL,EAAO,GAAkB,wBAAyB,GAClD,EAAO,GAAkB,uBAAwB,GAGvD,SACG,KACC,EAAO,IACP,GAAK,IAEJ,UAAU,IAAM,CACf,GAAsB,KAI5B,EACG,KACC,EAAU,GACV,GAAe,IAEd,UAAU,CAAC,CAAC,CAAE,SAAS,CAAE,YAAa,CACrC,AAAI,EACF,GAAoB,EAAM,EAAM,QAEhC,GAAsB,KAI9B,EACG,KACC,EAAU,GACV,EAAI,IAAM,GAAsB,IAChC,EAAU,CAAC,CAAE,WAAY,EACvB,EAAG,GAAG,EAAM,MAAM,EAAG,KACrB,EAAG,GAAG,EAAM,MAAM,KACf,KACC,GAAY,GACZ,GAAQ,GACR,EAAU,CAAC,CAAC,KAAW,EAAG,GAAG,QAIlC,UAAU,GAAU,CACnB,GAAsB,EAAM,GAAuB,MAWlD,AAPS,EACb,KACC,EAAO,IACP,EAAI,CAAC,CAAE,UAAW,IAKnB,KACC,EAAI,GACJ,EAAI,GAAU,GAAE,IAAK,GAAO,KChF3B,YACL,EAAkB,CAAE,UACK,CACzB,MAAO,GACJ,KACC,EAAI,CAAC,CAAE,WAAY,CACjB,GAAM,GAAM,KACZ,SAAI,KAAO,GACX,EAAI,aAAa,OAAO,KACxB,EAAI,aAAa,IAAI,IAAK,GACnB,CAAE,UAaV,YACL,EAAuB,EACa,CACpC,GAAM,GAAY,GAAI,GACtB,SAAU,UAAU,CAAC,CAAE,SAAU,CAC/B,EAAG,aAAa,sBAAuB,EAAG,MAC1C,EAAG,KAAO,GAAG,MAIf,EAAU,EAAI,SACX,UAAU,GAAM,EAAG,kBAGf,GAAiB,EAAI,GACzB,KACC,EAAI,GACJ,EAAI,GAAU,GAAE,IAAK,GAAO,KCjC3B,YACL,EAAiB,CAAE,OAAqB,CAAE,aACJ,CACtC,GAAM,GAAY,GAAI,GAGhB,EAAS,GAAoB,gBAC7B,EAAS,EAAU,EAAO,WAC7B,KACC,EAAU,IACV,EAAI,IAAM,EAAM,OAChB,KAIJ,SACG,KACC,GAAkB,GAClB,EAAI,CAAC,CAAC,CAAE,eAAe,KAAW,CAChC,GAAM,GAAQ,EAAM,MAAM,YAC1B,GAAI,kBAAa,SAAU,EAAM,EAAM,OAAS,GAAI,CAClD,GAAM,GAAO,EAAY,EAAY,OAAS,GAC9C,AAAI,EAAK,WAAW,EAAM,EAAM,OAAS,KACvC,GAAM,EAAM,OAAS,GAAK,OAE5B,GAAM,OAAS,EAEjB,MAAO,MAGR,UAAU,GAAS,EAAG,UAAY,EAChC,KAAK,IACL,QAAQ,MAAO,WAItB,EACG,KACC,EAAO,CAAC,CAAE,UAAW,IAAS,WAE7B,UAAU,GAAO,CAChB,OAAQ,EAAI,UAGL,aACH,AACE,EAAG,UAAU,QACb,EAAM,iBAAmB,EAAM,MAAM,QAErC,GAAM,MAAQ,EAAG,WACnB,SAYH,AAPS,EACb,KACC,EAAO,IACP,EAAI,CAAC,CAAE,UAAW,IAKnB,KACC,EAAI,GACJ,EAAI,IAAO,EAAE,IAAK,MCvDjB,YACL,EAAiB,CAAE,SAAQ,aACI,CAC/B,GAAM,GAAS,KACf,GAAI,CACF,GAAM,GAAS,GAAkB,EAAO,OAAQ,GAG1C,EAAS,GAAoB,eAAgB,GAC7C,EAAS,GAAoB,gBAAiB,GAG9C,CAAE,MAAK,OAAQ,EACrB,EACG,KACC,EAAO,IACP,GAAO,EACJ,KACC,EAAO,IACP,GAAK,MAIR,UAAU,EAAI,KAAK,KAAK,IAG7B,EACG,KACC,EAAO,CAAC,CAAE,UAAW,IAAS,WAE7B,UAAU,GAAO,CAChB,GAAM,GAAS,KACf,OAAQ,EAAI,UAGL,QACH,GAAI,IAAW,EAAO,CACpB,GAAM,GAAU,GAAI,KACpB,OAAW,KAAU,GACnB,sBAAuB,GACtB,CACD,GAAM,GAAU,EAAO,kBACvB,EAAQ,IAAI,EAAQ,WAClB,EAAQ,aAAa,mBAKzB,GAAI,EAAQ,KAAM,CAChB,GAAM,CAAC,CAAC,IAAS,CAAC,GAAG,GAAS,KAAK,CAAC,CAAC,CAAE,GAAI,CAAC,CAAE,KAAO,EAAI,GACzD,EAAK,QAIP,EAAI,QAEN,UAGG,aACA,MACH,GAAU,SAAU,IACpB,GAAgB,EAAO,IACvB,UAGG,cACA,YACH,GAAI,MAAO,IAAW,YACpB,GAAgB,OACX,CACL,GAAM,GAAM,CAAC,EAAO,GAAG,EACrB,wDACA,IAEI,EAAI,KAAK,IAAI,EACjB,MAAK,IAAI,EAAG,EAAI,QAAQ,IAAW,EAAI,OACrC,GAAI,OAAS,UAAY,GAAK,IAE9B,EAAI,QACR,GAAgB,EAAI,IAItB,EAAI,QACJ,cAIA,AAAI,IAAU,MACZ,GAAgB,MAK5B,EACG,KACC,EAAO,CAAC,CAAE,UAAW,IAAS,WAE7B,UAAU,GAAO,CAChB,OAAQ,EAAI,UAGL,QACA,QACA,IACH,GAAgB,GAChB,GAAoB,GACpB,EAAI,QACJ,SAKV,GAAM,GAAU,GAAiB,EAAO,GAClC,EAAU,GAAkB,EAAQ,EAAQ,CAAE,WACpD,MAAO,GAAM,EAAQ,GAClB,KACC,GAGE,GAAG,GAAqB,eAAgB,GACvC,IAAI,GAAS,GAAiB,EAAO,CAAE,YAGxC,GAAG,GAAqB,iBAAkB,GACvC,IAAI,GAAS,GAAmB,EAAO,EAAQ,CAAE,uBAKnD,EAAP,CACA,SAAG,OAAS,GACL,GCxJJ,YACL,EAAiB,CAAE,SAAQ,aACa,CACxC,MAAO,GAAc,CACnB,EACA,EACG,KACC,EAAU,MACV,EAAO,GAAO,EAAI,aAAa,IAAI,SAGtC,KACC,EAAI,CAAC,CAAC,EAAO,KAAS,GAAuB,EAAM,QACjD,EAAI,aAAa,IAAI,OAEvB,EAAI,GAAM,CAxFhB,MAyFQ,GAAM,GAAQ,GAAI,KAGZ,EAAK,SAAS,mBAAmB,EAAI,WAAW,WACtD,OAAS,GAAO,EAAG,WAAY,EAAM,EAAO,EAAG,WAC7C,GAAI,KAAK,gBAAL,cAAoB,aAAc,CACpC,GAAM,GAAW,EAAK,YAChB,EAAW,EAAG,GACpB,AAAI,EAAS,OAAS,EAAS,QAC7B,EAAM,IAAI,EAAmB,GAKnC,OAAW,CAAC,EAAM,IAAS,GAAO,CAChC,GAAM,CAAE,cAAe,EAAE,OAAQ,KAAM,GACvC,EAAK,YAAY,GAAG,MAAM,KAAK,IAIjC,MAAO,CAAE,IAAK,EAAI,YCXnB,YACL,EAAiB,CAAE,YAAW,SACT,CACrB,GAAM,GACJ,EAAG,cAAe,UAClB,EAAG,cAAe,cAAe,UAGnC,MAAO,GAAc,CAAC,EAAO,IAC1B,KACC,EAAI,CAAC,CAAC,CAAE,SAAQ,UAAU,CAAE,OAAQ,CAAE,SACpC,GAAS,EACL,KAAK,IAAI,EAAQ,KAAK,IAAI,EAAG,EAAI,IACjC,EACG,CACL,SACA,OAAQ,GAAK,EAAS,KAG1B,EAAqB,CAAC,EAAG,IACvB,EAAE,SAAW,EAAE,QACf,EAAE,SAAW,EAAE,SAahB,YACL,EAAiB,EACe,CADf,QAAE,YAAF,EAAc,KAAd,EAAc,CAAZ,YAEnB,GAAM,GAAY,GAAI,GACtB,SACG,KACC,EAAU,GACV,GAAe,IAEd,UAAU,CAGT,KAAK,CAAC,CAAE,UAAU,CAAE,OAAQ,IAAW,CACrC,GAAiB,EAAI,GACrB,GAAiB,EAAI,IAIvB,UAAW,CACT,GAAmB,GACnB,GAAmB,MAKpB,GAAa,EAAI,GACrB,KACC,EAAI,GACJ,EAAI,GAAU,GAAE,IAAK,GAAO,KC3G3B,YACL,EAAc,EACW,CACzB,GAAI,MAAO,IAAS,YAAa,CAC/B,GAAM,GAAM,gCAAgC,KAAQ,IACpD,MAAO,IAGL,GAAqB,GAAG,qBACrB,KACC,EAAI,GAAY,EACd,QAAS,EAAQ,YAEnB,GAAe,KAInB,GAAkB,GACf,KACC,EAAI,GAAS,EACX,MAAO,EAAK,iBACZ,MAAO,EAAK,eAEd,GAAe,MAGlB,KACC,EAAI,CAAC,CAAC,EAAS,KAAW,OAAK,GAAY,SAI1C,CACL,GAAM,GAAM,gCAAgC,IAC5C,MAAO,IAAkB,GACtB,KACC,EAAI,GAAS,EACX,aAAc,EAAK,gBAErB,GAAe,MCjDhB,YACL,EAAc,EACW,CACzB,GAAM,GAAM,WAAW,qBAAwB,mBAAmB,KAClE,MAAO,IAA2B,GAC/B,KACC,EAAI,CAAC,CAAE,aAAY,iBAAmB,EACpC,MAAO,EACP,MAAO,KAET,GAAe,KCed,YACL,EACyB,CACzB,GAAM,CAAC,GAAQ,EAAI,MAAM,sBAAwB,GACjD,OAAQ,EAAK,mBAGN,SACH,GAAM,CAAC,CAAE,EAAM,GAAQ,EAAI,MAAM,uCACjC,MAAO,IAA2B,EAAM,OAGrC,SACH,GAAM,CAAC,CAAE,EAAM,GAAQ,EAAI,MAAM,sCACjC,MAAO,IAA2B,EAAM,WAIxC,MAAO,IC9Bb,GAAI,IAgBG,YACL,EACoB,CACpB,MAAO,SAAW,GAAM,IAAM,CAC5B,GAAM,GAAO,eAAe,QAAQ,SAAS,aAC7C,GAAI,EACF,MAAO,GAAgB,KAAK,MAAM,IAC7B,CACL,GAAM,GAAS,GAAiB,EAAG,MACnC,SAAO,UAAU,GAAS,CACxB,GAAI,CACF,eAAe,QAAQ,SAAS,YAAa,KAAK,UAAU,UACrD,EAAP,KAMG,KAGR,KACC,GAAW,IAAM,GACjB,EAAO,GAAS,OAAO,KAAK,GAAO,OAAS,GAC5C,EAAI,GAAU,EAAE,WAChB,GAAY,KAWX,YACL,EAC+B,CAC/B,GAAM,GAAY,GAAI,GACtB,SAAU,UAAU,CAAC,CAAE,WAAY,CACjC,GAAe,EAAI,GAAkB,IACrC,GAAe,EAAI,UAId,GAAY,GAChB,KACC,EAAI,GACJ,EAAI,GAAU,GAAE,IAAK,GAAO,KCpC3B,YACL,EAAiB,CAAE,YAAW,WACZ,CAClB,MAAO,IAAiB,SAAS,MAC9B,KACC,EAAU,IAAM,GAAgB,EAAI,CAAE,UAAS,eAC/C,EAAI,CAAC,CAAE,OAAQ,CAAE,QACR,EACL,OAAQ,GAAK,MAGjB,EAAwB,WAevB,YACL,EAAiB,EACY,CAC7B,GAAM,GAAY,GAAI,GACtB,SACG,KACC,EAAU,IAET,UAAU,CAGT,KAAK,CAAE,UAAU,CACf,AAAI,EACF,GAAa,EAAI,UAEjB,GAAe,IAInB,UAAW,CACT,GAAe,MAKhB,GAAU,EAAI,GAClB,KACC,EAAI,GACJ,EAAI,GAAU,GAAE,IAAK,GAAO,KC1B3B,YACL,EAA8B,CAAE,YAAW,WACd,CAC7B,GAAM,GAAQ,GAAI,KAClB,OAAW,KAAU,GAAS,CAC5B,GAAM,GAAK,mBAAmB,EAAO,KAAK,UAAU,IAC9C,EAAS,GAAW,QAAQ,OAClC,AAAI,MAAO,IAAW,aACpB,EAAM,IAAI,EAAQ,GAItB,GAAM,GAAU,EACb,KACC,EAAI,GAAU,GAAK,EAAO,SA4E9B,MAAO,AAxEY,IAAiB,SAAS,MAC1C,KACC,EAAwB,UAGxB,EAAI,IAAM,CACR,GAAI,GAA4B,GAChC,MAAO,CAAC,GAAG,GAAO,OAAO,CAAC,EAAO,CAAC,EAAQ,KAAY,CACpD,KAAO,EAAK,QAEN,AADS,EAAM,IAAI,EAAK,EAAK,OAAS,IACjC,SAAW,EAAO,SACzB,EAAK,MAOT,GAAI,GAAS,EAAO,UACpB,KAAO,CAAC,GAAU,EAAO,eACvB,EAAS,EAAO,cAChB,EAAS,EAAO,UAIlB,MAAO,GAAM,IACX,CAAC,GAAG,EAAO,CAAC,GAAG,EAAM,IAAS,UAC9B,IAED,GAAI,QAIT,EAAI,GAAS,GAAI,KAAI,CAAC,GAAG,GAAO,KAAK,CAAC,CAAC,CAAE,GAAI,CAAC,CAAE,KAAO,EAAI,KAG3D,EAAU,GAAS,EAAc,CAAC,EAAS,IACxC,KACC,GAAK,CAAC,CAAC,EAAM,GAAO,CAAC,EAAQ,CAAE,OAAQ,CAAE,SAAW,CAGlD,KAAO,EAAK,QAAQ,CAClB,GAAM,CAAC,CAAE,GAAU,EAAK,GACxB,GAAI,EAAS,EAAS,EACpB,EAAO,CAAC,GAAG,EAAM,EAAK,aAEtB,OAKJ,KAAO,EAAK,QAAQ,CAClB,GAAM,CAAC,CAAE,GAAU,EAAK,EAAK,OAAS,GACtC,GAAI,EAAS,GAAU,EACrB,EAAO,CAAC,EAAK,MAAQ,GAAG,OAExB,OAKJ,MAAO,CAAC,EAAM,IACb,CAAC,GAAI,CAAC,GAAG,KACZ,EAAqB,CAAC,EAAG,IACvB,EAAE,KAAO,EAAE,IACX,EAAE,KAAO,EAAE,OAQlB,KACC,EAAI,CAAC,CAAC,EAAM,KAAW,EACrB,KAAM,EAAK,IAAI,CAAC,CAAC,KAAU,GAC3B,KAAM,EAAK,IAAI,CAAC,CAAC,KAAU,MAI7B,EAAU,CAAE,KAAM,GAAI,KAAM,KAC5B,GAAY,EAAG,GACf,EAAI,CAAC,CAAC,EAAG,KAGH,EAAE,KAAK,OAAS,EAAE,KAAK,OAClB,CACL,KAAM,EAAE,KAAK,MAAM,KAAK,IAAI,EAAG,EAAE,KAAK,OAAS,GAAI,EAAE,KAAK,QAC1D,KAAM,IAKD,CACL,KAAM,EAAE,KAAK,MAAM,IACnB,KAAM,EAAE,KAAK,MAAM,EAAG,EAAE,KAAK,OAAS,EAAE,KAAK,WAiBlD,YACL,EAAiB,EACuB,CACxC,GAAM,GAAY,GAAI,GACtB,EACG,KACC,EAAU,IAET,UAAU,CAAC,CAAE,OAAM,UAAW,CAG7B,OAAW,CAAC,IAAW,GACrB,GAAkB,GAClB,GAAiB,GAInB,OAAW,CAAC,EAAO,CAAC,KAAY,GAAK,UACnC,GAAgB,EAAQ,IAAU,EAAK,OAAS,GAChD,GAAe,EAAQ,UAK/B,GAAM,GAAU,EAA+B,cAAe,GAC9D,MAAO,IAAqB,EAAS,GAClC,KACC,EAAI,GACJ,EAAI,GAAU,GAAE,IAAK,GAAO,KC/K3B,YACL,EAAkB,CAAE,YAAW,SACR,CAGvB,GAAM,GAAa,EAChB,KACC,EAAI,CAAC,CAAE,OAAQ,CAAE,QAAU,GAC3B,GAAY,EAAG,GACf,EAAI,CAAC,CAAC,EAAG,KAAO,EAAI,GAAK,GACzB,KAIE,EAAU,EACb,KACC,EAAwB,WAI5B,MAAO,GAAc,CAAC,EAAS,IAC5B,KACC,EAAI,CAAC,CAAC,CAAE,UAAU,KAAgB,EAChC,OAAQ,CAAE,IAAU,MAEtB,EAAqB,CAAC,EAAG,IACvB,EAAE,SAAW,EAAE,SAehB,YACL,EAAiB,CAAE,YAAW,UAAS,SACL,CAClC,GAAM,GAAY,GAAI,GACtB,SACG,KACC,EAAU,GACV,GAAe,EACZ,KACC,EAAwB,aAI3B,UAAU,CAGT,KAAK,CAAC,CAAE,UAAU,CAAE,WAAW,CAC7B,GAAmB,EAAI,EAAS,IAChC,AAAI,EACF,IAAkB,EAAI,UACtB,GAAgB,EAAI,KAEpB,GAAoB,IAKxB,UAAW,CACT,GAAqB,GACrB,GAAoB,MAKrB,GAAe,EAAI,CAAE,YAAW,UAAS,UAC7C,KACC,EAAI,GACJ,EAAI,GAAU,GAAE,IAAK,GAAO,KCnH3B,YACL,CAAE,YAAW,WACP,CACN,EACG,KACC,EAAU,IAAM,EAAG,GAAG,EACpB,mCAEF,EAAI,GAAM,CACR,EAAG,cAAgB,GACnB,EAAG,QAAU,KAEf,GAAS,GAAM,EAAU,EAAI,UAC1B,KACC,GAAU,IAAM,EAAG,aAAa,kBAChC,GAAM,KAGV,GAAe,IAEd,UAAU,CAAC,CAAC,EAAI,KAAY,CAC3B,EAAG,gBAAgB,iBACf,GACF,GAAG,QAAU,MC5BvB,aAAkC,CAChC,MAAO,qBAAqB,KAAK,UAAU,WAkBtC,YACL,CAAE,aACI,CACN,EACG,KACC,EAAU,IAAM,EAAG,GAAG,EAAY,yBAClC,EAAI,GAAM,EAAG,gBAAgB,sBAC7B,EAAO,IACP,GAAS,GAAM,EAAU,EAAI,cAC1B,KACC,GAAM,MAIT,UAAU,GAAM,CACf,GAAM,GAAM,EAAG,UAGf,AAAI,IAAQ,EACV,EAAG,UAAY,EAGN,EAAM,EAAG,eAAiB,EAAG,cACtC,GAAG,UAAY,EAAM,KC9BxB,YACL,CAAE,YAAW,WACP,CACN,EAAc,CAAC,GAAY,UAAW,IACnC,KACC,EAAI,CAAC,CAAC,EAAQ,KAAY,GAAU,CAAC,GACrC,EAAU,GAAU,EAAG,GACpB,KACC,GAAM,EAAS,IAAM,KACrB,EAAU,KAGd,GAAe,IAEd,UAAU,CAAC,CAAC,EAAQ,CAAE,OAAQ,CAAE,SAAU,CACzC,AAAI,EACF,GAAc,SAAS,KAAM,GAE7B,GAAgB,SAAS,QrLDnC,SAAS,gBAAgB,UAAU,OAAO,SAC1C,SAAS,gBAAgB,UAAU,IAAI,MAGvC,GAAM,IAAY,KACZ,GAAY,KACZ,GAAY,KACZ,GAAY,KAGZ,GAAY,KACZ,GAAY,GAAW,sBACvB,GAAY,GAAW,uBACvB,GAAY,KAGZ,GAAS,KACT,GAAS,SAAS,MAAM,UAAU,UACpC,gCAAU,QAAS,GACnB,GAAI,KAAI,2BAA4B,GAAO,OAE3C,EAGE,GAAS,GAAI,GACnB,GAAiB,CAAE,YAGnB,AAAI,GAAQ,uBACV,GAAoB,CAAE,aAAW,aAAW,eA/G9C,OAkHA,AAAI,QAAO,UAAP,eAAgB,YAAa,QAC/B,KAGF,EAAM,GAAW,IACd,KACC,GAAM,MAEL,UAAU,IAAM,CACf,GAAU,SAAU,IACpB,GAAU,SAAU,MAI1B,GACG,KACC,EAAO,CAAC,CAAE,UAAW,IAAS,WAE7B,UAAU,GAAO,CAChB,OAAQ,EAAI,UAGL,QACA,IACH,GAAM,GAAO,GAAW,oBACxB,AAAI,MAAO,IAAS,aAClB,EAAK,QACP,UAGG,QACA,IACH,GAAM,GAAO,GAAW,oBACxB,AAAI,MAAO,IAAS,aAClB,EAAK,QACP,SAKV,GAAmB,CAAE,aAAW,aAChC,GAAe,CAAE,eACjB,GAAgB,CAAE,aAAW,aAG7B,GAAM,IAAU,GAAY,GAAoB,UAAW,CAAE,eACvD,GAAQ,GACX,KACC,EAAI,IAAM,GAAoB,SAC9B,EAAU,GAAM,GAAU,EAAI,CAAE,aAAW,cAC3C,GAAY,IAIV,GAAW,EAGf,GAAG,GAAqB,UACrB,IAAI,GAAM,GAAY,EAAI,CAAE,aAG/B,GAAG,GAAqB,UACrB,IAAI,GAAM,GAAY,EAAI,CAAE,aAAW,WAAS,YAGnD,GAAG,GAAqB,WACrB,IAAI,GAAM,GAAa,IAG1B,GAAG,GAAqB,UACrB,IAAI,GAAM,GAAY,EAAI,CAAE,UAAQ,gBAGvC,GAAG,GAAqB,UACrB,IAAI,GAAM,GAAY,KAIrB,GAAW,GAAM,IAAM,EAG3B,GAAG,GAAqB,WACrB,IAAI,GAAM,GAAa,EAAI,CAAE,WAAS,aAAW,aAGpD,GAAG,GAAqB,WACrB,IAAI,GAAM,GAAQ,oBACf,GAAoB,EAAI,CAAE,UAAQ,eAClC,GAIN,GAAG,GAAqB,gBACrB,IAAI,GAAM,GAAiB,EAAI,CAAE,aAAW,cAG/C,GAAG,GAAqB,WACrB,IAAI,GAAM,EAAG,aAAa,kBAAoB,aAC3C,GAAG,GAAS,IAAM,GAAa,EAAI,CAAE,aAAW,WAAS,YACzD,GAAG,GAAS,IAAM,GAAa,EAAI,CAAE,aAAW,WAAS,aAI/D,GAAG,GAAqB,QACrB,IAAI,GAAM,GAAU,EAAI,CAAE,aAAW,cAGxC,GAAG,GAAqB,OACrB,IAAI,GAAM,GAAqB,EAAI,CAAE,aAAW,cAGnD,GAAG,GAAqB,OACrB,IAAI,GAAM,GAAe,EAAI,CAAE,aAAW,WAAS,cAIlD,GAAa,GAChB,KACC,EAAU,IAAM,IAChB,GAAU,IACV,GAAY,IAIhB,GAAW,YAMX,OAAO,UAAa,GACpB,OAAO,UAAa,GACpB,OAAO,QAAa,GACpB,OAAO,UAAa,GACpB,OAAO,UAAa,GACpB,OAAO,QAAa,GACpB,OAAO,QAAa,GACpB,OAAO,OAAa,GACpB,OAAO,OAAa,GACpB,OAAO,WAAa",
-  "names": []
-}
diff --git a/assets/javascripts/lunr/min/lunr.el.min.js b/assets/javascripts/lunr/min/lunr.el.min.js
new file mode 100644
index 000000000..ace017bd6
--- /dev/null
+++ b/assets/javascripts/lunr/min/lunr.el.min.js
@@ -0,0 +1 @@
+!function(e,t){"function"==typeof define&&define.amd?define(t):"object"==typeof exports?module.exports=t():t()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.el=function(){this.pipeline.reset(),void 0===this.searchPipeline&&this.pipeline.add(e.el.trimmer,e.el.normilizer),this.pipeline.add(e.el.stopWordFilter,e.el.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.el.stemmer))},e.el.wordCharacters="A-Za-zΑαΒβΓγΔδΕεΖζΗηΘθΙιΚκΛλΜμΝνΞξΟοΠπΡρΣσςΤτΥυΦφΧχΨψΩωΆάΈέΉήΊίΌόΎύΏώΪΐΫΰΐΰ",e.el.trimmer=e.trimmerSupport.generateTrimmer(e.el.wordCharacters),e.Pipeline.registerFunction(e.el.trimmer,"trimmer-el"),e.el.stemmer=function(){function e(e){return s.test(e)}function t(e){return/[ΑΕΗΙΟΥΩ]$/.test(e)}function r(e){return/[ΑΕΗΙΟΩ]$/.test(e)}function n(n){var s=n;if(n.length<3)return s;if(!e(n))return s;if(i.indexOf(n)>=0)return s;var u=new RegExp("(.*)("+Object.keys(l).join("|")+")$"),o=u.exec(s);return null!==o&&(s=o[1]+l[o[2]]),null!==(o=/^(.+?)(ΑΔΕΣ|ΑΔΩΝ)$/.exec(s))&&(s=o[1],/(ΟΚ|ΜΑΜ|ΜΑΝ|ΜΠΑΜΠ|ΠΑΤΕΡ|ΓΙΑΓΙ|ΝΤΑΝΤ|ΚΥΡ|ΘΕΙ|ΠΕΘΕΡ|ΜΟΥΣΑΜ|ΚΑΠΛΑΜ|ΠΑΡ|ΨΑΡ|ΤΖΟΥΡ|ΤΑΜΠΟΥΡ|ΓΑΛΑΤ|ΦΑΦΛΑΤ)$/.test(o[1])||(s+="ΑΔ")),null!==(o=/^(.+?)(ΕΔΕΣ|ΕΔΩΝ)$/.exec(s))&&(s=o[1],/(ΟΠ|ΙΠ|ΕΜΠ|ΥΠ|ΓΗΠ|ΔΑΠ|ΚΡΑΣΠ|ΜΙΛ)$/.test(o[1])&&(s+="ΕΔ")),null!==(o=/^(.+?)(ΟΥΔΕΣ|ΟΥΔΩΝ)$/.exec(s))&&(s=o[1],/(ΑΡΚ|ΚΑΛΙΑΚ|ΠΕΤΑΛ|ΛΙΧ|ΠΛΕΞ|ΣΚ|Σ|ΦΛ|ΦΡ|ΒΕΛ|ΛΟΥΛ|ΧΝ|ΣΠ|ΤΡΑΓ|ΦΕ)$/.test(o[1])&&(s+="ΟΥΔ")),null!==(o=/^(.+?)(ΕΩΣ|ΕΩΝ|ΕΑΣ|ΕΑ)$/.exec(s))&&(s=o[1],/^(Θ|Δ|ΕΛ|ΓΑΛ|Ν|Π|ΙΔ|ΠΑΡ|ΣΤΕΡ|ΟΡΦ|ΑΝΔΡ|ΑΝΤΡ)$/.test(o[1])&&(s+="Ε")),null!==(o=/^(.+?)(ΕΙΟ|ΕΙΟΣ|ΕΙΟΙ|ΕΙΑ|ΕΙΑΣ|ΕΙΕΣ|ΕΙΟΥ|ΕΙΟΥΣ|ΕΙΩΝ)$/.exec(s))&&o[1].length>4&&(s=o[1]),null!==(o=/^(.+?)(ΙΟΥΣ|ΙΑΣ|ΙΕΣ|ΙΟΣ|ΙΟΥ|ΙΟΙ|ΙΩΝ|ΙΟΝ|ΙΑ|ΙΟ)$/.exec(s))&&(s=o[1],(t(s)||s.length<2||/^(ΑΓ|ΑΓΓΕΛ|ΑΓΡ|ΑΕΡ|ΑΘΛ|ΑΚΟΥΣ|ΑΞ|ΑΣ|Β|ΒΙΒΛ|ΒΥΤ|Γ|ΓΙΑΓ|ΓΩΝ|Δ|ΔΑΝ|ΔΗΛ|ΔΗΜ|ΔΟΚΙΜ|ΕΛ|ΖΑΧΑΡ|ΗΛ|ΗΠ|ΙΔ|ΙΣΚ|ΙΣΤ|ΙΟΝ|ΙΩΝ|ΚΙΜΩΛ|ΚΟΛΟΝ|ΚΟΡ|ΚΤΗΡ|ΚΥΡ|ΛΑΓ|ΛΟΓ|ΜΑΓ|ΜΠΑΝ|ΜΠΡ|ΝΑΥΤ|ΝΟΤ|ΟΠΑΛ|ΟΞ|ΟΡ|ΟΣ|ΠΑΝΑΓ|ΠΑΤΡ|ΠΗΛ|ΠΗΝ|ΠΛΑΙΣ|ΠΟΝΤ|ΡΑΔ|ΡΟΔ|ΣΚ|ΣΚΟΡΠ|ΣΟΥΝ|ΣΠΑΝ|ΣΤΑΔ|ΣΥΡ|ΤΗΛ|ΤΙΜ|ΤΟΚ|ΤΟΠ|ΤΡΟΧ|ΦΙΛ|ΦΩΤ|Χ|ΧΙΛ|ΧΡΩΜ|ΧΩΡ)$/.test(o[1]))&&(s+="Ι"),/^(ΠΑΛ)$/.test(o[1])&&(s+="ΑΙ")),null!==(o=/^(.+?)(ΙΚΟΣ|ΙΚΟΝ|ΙΚΕΙΣ|ΙΚΟΙ|ΙΚΕΣ|ΙΚΟΥΣ|ΙΚΗ|ΙΚΗΣ|ΙΚΟ|ΙΚΑ|ΙΚΟΥ|ΙΚΩΝ|ΙΚΩΣ)$/.exec(s))&&(s=o[1],(t(s)||/^(ΑΔ|ΑΛ|ΑΜΑΝ|ΑΜΕΡ|ΑΜΜΟΧΑΛ|ΑΝΗΘ|ΑΝΤΙΔ|ΑΠΛ|ΑΤΤ|ΑΦΡ|ΒΑΣ|ΒΡΩΜ|ΓΕΝ|ΓΕΡ|Δ|ΔΙΚΑΝ|ΔΥΤ|ΕΙΔ|ΕΝΔ|ΕΞΩΔ|ΗΘ|ΘΕΤ|ΚΑΛΛΙΝ|ΚΑΛΠ|ΚΑΤΑΔ|ΚΟΥΖΙΝ|ΚΡ|ΚΩΔ|ΛΟΓ|Μ|ΜΕΡ|ΜΟΝΑΔ|ΜΟΥΛ|ΜΟΥΣ|ΜΠΑΓΙΑΤ|ΜΠΑΝ|ΜΠΟΛ|ΜΠΟΣ|ΜΥΣΤ|Ν|ΝΙΤ|ΞΙΚ|ΟΠΤ|ΠΑΝ|ΠΕΤΣ|ΠΙΚΑΝΤ|ΠΙΤΣ|ΠΛΑΣΤ|ΠΛΙΑΤΣ|ΠΟΝΤ|ΠΟΣΤΕΛΝ|ΠΡΩΤΟΔ|ΣΕΡΤ|ΣΗΜΑΝΤ|ΣΤΑΤ|ΣΥΝΑΔ|ΣΥΝΟΜΗΛ|ΤΕΛ|ΤΕΧΝ|ΤΡΟΠ|ΤΣΑΜ|ΥΠΟΔ|Φ|ΦΙΛΟΝ|ΦΥΛΟΔ|ΦΥΣ|ΧΑΣ)$/.test(o[1])||/(ΦΟΙΝ)$/.test(o[1]))&&(s+="ΙΚ")),"ΑΓΑΜΕ"===s&&(s="ΑΓΑΜ"),null!==(o=/^(.+?)(ΑΓΑΜΕ|ΗΣΑΜΕ|ΟΥΣΑΜΕ|ΗΚΑΜΕ|ΗΘΗΚΑΜΕ)$/.exec(s))&&(s=o[1]),null!==(o=/^(.+?)(ΑΜΕ)$/.exec(s))&&(s=o[1],/^(ΑΝΑΠ|ΑΠΟΘ|ΑΠΟΚ|ΑΠΟΣΤ|ΒΟΥΒ|ΞΕΘ|ΟΥΛ|ΠΕΘ|ΠΙΚΡ|ΠΟΤ|ΣΙΧ|Χ)$/.test(o[1])&&(s+="ΑΜ")),null!==(o=/^(.+?)(ΑΓΑΝΕ|ΗΣΑΝΕ|ΟΥΣΑΝΕ|ΙΟΝΤΑΝΕ|ΙΟΤΑΝΕ|ΙΟΥΝΤΑΝΕ|ΟΝΤΑΝΕ|ΟΤΑΝΕ|ΟΥΝΤΑΝΕ|ΗΚΑΝΕ|ΗΘΗΚΑΝΕ)$/.exec(s))&&(s=o[1],/^(ΤΡ|ΤΣ)$/.test(o[1])&&(s+="ΑΓΑΝ")),null!==(o=/^(.+?)(ΑΝΕ)$/.exec(s))&&(s=o[1],(r(s)||/^(ΒΕΤΕΡ|ΒΟΥΛΚ|ΒΡΑΧΜ|Γ|ΔΡΑΔΟΥΜ|Θ|ΚΑΛΠΟΥΖ|ΚΑΣΤΕΛ|ΚΟΡΜΟΡ|ΛΑΟΠΛ|ΜΩΑΜΕΘ|Μ|ΜΟΥΣΟΥΛΜΑΝ|ΟΥΛ|Π|ΠΕΛΕΚ|ΠΛ|ΠΟΛΙΣ|ΠΟΡΤΟΛ|ΣΑΡΑΚΑΤΣ|ΣΟΥΛΤ|ΤΣΑΡΛΑΤ|ΟΡΦ|ΤΣΙΓΓ|ΤΣΟΠ|ΦΩΤΟΣΤΕΦ|Χ|ΨΥΧΟΠΛ|ΑΓ|ΟΡΦ|ΓΑΛ|ΓΕΡ|ΔΕΚ|ΔΙΠΛ|ΑΜΕΡΙΚΑΝ|ΟΥΡ|ΠΙΘ|ΠΟΥΡΙΤ|Σ|ΖΩΝΤ|ΙΚ|ΚΑΣΤ|ΚΟΠ|ΛΙΧ|ΛΟΥΘΗΡ|ΜΑΙΝΤ|ΜΕΛ|ΣΙΓ|ΣΠ|ΣΤΕΓ|ΤΡΑΓ|ΤΣΑΓ|Φ|ΕΡ|ΑΔΑΠ|ΑΘΙΓΓ|ΑΜΗΧ|ΑΝΙΚ|ΑΝΟΡΓ|ΑΠΗΓ|ΑΠΙΘ|ΑΤΣΙΓΓ|ΒΑΣ|ΒΑΣΚ|ΒΑΘΥΓΑΛ|ΒΙΟΜΗΧ|ΒΡΑΧΥΚ|ΔΙΑΤ|ΔΙΑΦ|ΕΝΟΡΓ|ΘΥΣ|ΚΑΠΝΟΒΙΟΜΗΧ|ΚΑΤΑΓΑΛ|ΚΛΙΒ|ΚΟΙΛΑΡΦ|ΛΙΒ|ΜΕΓΛΟΒΙΟΜΗΧ|ΜΙΚΡΟΒΙΟΜΗΧ|ΝΤΑΒ|ΞΗΡΟΚΛΙΒ|ΟΛΙΓΟΔΑΜ|ΟΛΟΓΑΛ|ΠΕΝΤΑΡΦ|ΠΕΡΗΦ|ΠΕΡΙΤΡ|ΠΛΑΤ|ΠΟΛΥΔΑΠ|ΠΟΛΥΜΗΧ|ΣΤΕΦ|ΤΑΒ|ΤΕΤ|ΥΠΕΡΗΦ|ΥΠΟΚΟΠ|ΧΑΜΗΛΟΔΑΠ|ΨΗΛΟΤΑΒ)$/.test(o[1]))&&(s+="ΑΝ")),null!==(o=/^(.+?)(ΗΣΕΤΕ)$/.exec(s))&&(s=o[1]),null!==(o=/^(.+?)(ΕΤΕ)$/.exec(s))&&(s=o[1],(r(s)||/(ΟΔ|ΑΙΡ|ΦΟΡ|ΤΑΘ|ΔΙΑΘ|ΣΧ|ΕΝΔ|ΕΥΡ|ΤΙΘ|ΥΠΕΡΘ|ΡΑΘ|ΕΝΘ|ΡΟΘ|ΣΘ|ΠΥΡ|ΑΙΝ|ΣΥΝΔ|ΣΥΝ|ΣΥΝΘ|ΧΩΡ|ΠΟΝ|ΒΡ|ΚΑΘ|ΕΥΘ|ΕΚΘ|ΝΕΤ|ΡΟΝ|ΑΡΚ|ΒΑΡ|ΒΟΛ|ΩΦΕΛ)$/.test(o[1])||/^(ΑΒΑΡ|ΒΕΝ|ΕΝΑΡ|ΑΒΡ|ΑΔ|ΑΘ|ΑΝ|ΑΠΛ|ΒΑΡΟΝ|ΝΤΡ|ΣΚ|ΚΟΠ|ΜΠΟΡ|ΝΙΦ|ΠΑΓ|ΠΑΡΑΚΑΛ|ΣΕΡΠ|ΣΚΕΛ|ΣΥΡΦ|ΤΟΚ|Υ|Δ|ΕΜ|ΘΑΡΡ|Θ)$/.test(o[1]))&&(s+="ΕΤ")),null!==(o=/^(.+?)(ΟΝΤΑΣ|ΩΝΤΑΣ)$/.exec(s))&&(s=o[1],/^ΑΡΧ$/.test(o[1])&&(s+="ΟΝΤ"),/ΚΡΕ$/.test(o[1])&&(s+="ΩΝΤ")),null!==(o=/^(.+?)(ΟΜΑΣΤΕ|ΙΟΜΑΣΤΕ)$/.exec(s))&&(s=o[1],/^ΟΝ$/.test(o[1])&&(s+="ΟΜΑΣΤ")),null!==(o=/^(.+?)(ΙΕΣΤΕ)$/.exec(s))&&(s=o[1],/^(Π|ΑΠ|ΣΥΜΠ|ΑΣΥΜΠ|ΑΚΑΤΑΠ|ΑΜΕΤΑΜΦ)$/.test(o[1])&&(s+="ΙΕΣΤ")),null!==(o=/^(.+?)(ΕΣΤΕ)$/.exec(s))&&(s=o[1],/^(ΑΛ|ΑΡ|ΕΚΤΕΛ|Ζ|Μ|Ξ|ΠΑΡΑΚΑΛ|ΠΡΟ|ΝΙΣ)$/.test(o[1])&&(s+="ΕΣΤ")),null!==(o=/^(.+?)(ΗΘΗΚΑ|ΗΘΗΚΕΣ|ΗΘΗΚΕ)$/.exec(s))&&(s=o[1]),null!==(o=/^(.+?)(ΗΚΑ|ΗΚΕΣ|ΗΚΕ)$/.exec(s))&&(s=o[1],(/(ΣΚΩΛ|ΣΚΟΥΛ|ΝΑΡΘ|ΣΦ|ΟΘ|ΠΙΘ)$/.test(o[1])||/^(ΔΙΑΘ|Θ|ΠΑΡΑΚΑΤΑΘ|ΠΡΟΣΘ|ΣΥΝΘ)$/.test(o[1]))&&(s+="ΗΚ")),null!==(o=/^(.+?)(ΟΥΣΑ|ΟΥΣΕΣ|ΟΥΣΕ)$/.exec(s))&&(s=o[1],(t(s)||/^(ΦΑΡΜΑΚ|ΧΑΔ|ΑΓΚ|ΑΝΑΡΡ|ΒΡΟΜ|ΕΚΛΙΠ|ΛΑΜΠΙΔ|ΛΕΧ|Μ|ΠΑΤ|Ρ|Λ|ΜΕΔ|ΜΕΣΑΖ|ΥΠΟΤΕΙΝ|ΑΜ|ΑΙΘ|ΑΝΗΚ|ΔΕΣΠΟΖ|ΕΝΔΙΑΦΕΡ)$/.test(o[1])||/(ΠΟΔΑΡ|ΒΛΕΠ|ΠΑΝΤΑΧ|ΦΡΥΔ|ΜΑΝΤΙΛ|ΜΑΛΛ|ΚΥΜΑΤ|ΛΑΧ|ΛΗΓ|ΦΑΓ|ΟΜ|ΠΡΩΤ)$/.test(o[1]))&&(s+="ΟΥΣ")),null!==(o=/^(.+?)(ΑΓΑ|ΑΓΕΣ|ΑΓΕ)$/.exec(s))&&(s=o[1],(/^(ΑΒΑΣΤ|ΠΟΛΥΦ|ΑΔΗΦ|ΠΑΜΦ|Ρ|ΑΣΠ|ΑΦ|ΑΜΑΛ|ΑΜΑΛΛΙ|ΑΝΥΣΤ|ΑΠΕΡ|ΑΣΠΑΡ|ΑΧΑΡ|ΔΕΡΒΕΝ|ΔΡΟΣΟΠ|ΞΕΦ|ΝΕΟΠ|ΝΟΜΟΤ|ΟΛΟΠ|ΟΜΟΤ|ΠΡΟΣΤ|ΠΡΟΣΩΠΟΠ|ΣΥΜΠ|ΣΥΝΤ|Τ|ΥΠΟΤ|ΧΑΡ|ΑΕΙΠ|ΑΙΜΟΣΤ|ΑΝΥΠ|ΑΠΟΤ|ΑΡΤΙΠ|ΔΙΑΤ|ΕΝ|ΕΠΙΤ|ΚΡΟΚΑΛΟΠ|ΣΙΔΗΡΟΠ|Λ|ΝΑΥ|ΟΥΛΑΜ|ΟΥΡ|Π|ΤΡ|Μ)$/.test(o[1])||/(ΟΦ|ΠΕΛ|ΧΟΡΤ|ΛΛ|ΣΦ|ΡΠ|ΦΡ|ΠΡ|ΛΟΧ|ΣΜΗΝ)$/.test(o[1])&&!/^(ΨΟΦ|ΝΑΥΛΟΧ)$/.test(o[1])||/(ΚΟΛΛ)$/.test(o[1]))&&(s+="ΑΓ")),null!==(o=/^(.+?)(ΗΣΕ|ΗΣΟΥ|ΗΣΑ)$/.exec(s))&&(s=o[1],/^(Ν|ΧΕΡΣΟΝ|ΔΩΔΕΚΑΝ|ΕΡΗΜΟΝ|ΜΕΓΑΛΟΝ|ΕΠΤΑΝ|Ι)$/.test(o[1])&&(s+="ΗΣ")),null!==(o=/^(.+?)(ΗΣΤΕ)$/.exec(s))&&(s=o[1],/^(ΑΣΒ|ΣΒ|ΑΧΡ|ΧΡ|ΑΠΛ|ΑΕΙΜΝ|ΔΥΣΧΡ|ΕΥΧΡ|ΚΟΙΝΟΧΡ|ΠΑΛΙΜΨ)$/.test(o[1])&&(s+="ΗΣΤ")),null!==(o=/^(.+?)(ΟΥΝΕ|ΗΣΟΥΝΕ|ΗΘΟΥΝΕ)$/.exec(s))&&(s=o[1],/^(Ν|Ρ|ΣΠΙ|ΣΤΡΑΒΟΜΟΥΤΣ|ΚΑΚΟΜΟΥΤΣ|ΕΞΩΝ)$/.test(o[1])&&(s+="ΟΥΝ")),null!==(o=/^(.+?)(ΟΥΜΕ|ΗΣΟΥΜΕ|ΗΘΟΥΜΕ)$/.exec(s))&&(s=o[1],/^(ΠΑΡΑΣΟΥΣ|Φ|Χ|ΩΡΙΟΠΛ|ΑΖ|ΑΛΛΟΣΟΥΣ|ΑΣΟΥΣ)$/.test(o[1])&&(s+="ΟΥΜ")),null!=(o=/^(.+?)(ΜΑΤΟΙ|ΜΑΤΟΥΣ|ΜΑΤΟ|ΜΑΤΑ|ΜΑΤΩΣ|ΜΑΤΩΝ|ΜΑΤΟΣ|ΜΑΤΕΣ|ΜΑΤΗ|ΜΑΤΗΣ|ΜΑΤΟΥ)$/.exec(s))&&(s=o[1]+"Μ",/^(ΓΡΑΜ)$/.test(o[1])?s+="Α":/^(ΓΕ|ΣΤΑ)$/.test(o[1])&&(s+="ΑΤ")),null!==(o=/^(.+?)(ΟΥΑ)$/.exec(s))&&(s=o[1]+"ΟΥ"),n.length===s.length&&null!==(o=/^(.+?)(Α|ΑΓΑΤΕ|ΑΓΑΝ|ΑΕΙ|ΑΜΑΙ|ΑΝ|ΑΣ|ΑΣΑΙ|ΑΤΑΙ|ΑΩ|Ε|ΕΙ|ΕΙΣ|ΕΙΤΕ|ΕΣΑΙ|ΕΣ|ΕΤΑΙ|Ι|ΙΕΜΑΙ|ΙΕΜΑΣΤΕ|ΙΕΤΑΙ|ΙΕΣΑΙ|ΙΕΣΑΣΤΕ|ΙΟΜΑΣΤΑΝ|ΙΟΜΟΥΝ|ΙΟΜΟΥΝΑ|ΙΟΝΤΑΝ|ΙΟΝΤΟΥΣΑΝ|ΙΟΣΑΣΤΑΝ|ΙΟΣΑΣΤΕ|ΙΟΣΟΥΝ|ΙΟΣΟΥΝΑ|ΙΟΤΑΝ|ΙΟΥΜΑ|ΙΟΥΜΑΣΤΕ|ΙΟΥΝΤΑΙ|ΙΟΥΝΤΑΝ|Η|ΗΔΕΣ|ΗΔΩΝ|ΗΘΕΙ|ΗΘΕΙΣ|ΗΘΕΙΤΕ|ΗΘΗΚΑΤΕ|ΗΘΗΚΑΝ|ΗΘΟΥΝ|ΗΘΩ|ΗΚΑΤΕ|ΗΚΑΝ|ΗΣ|ΗΣΑΝ|ΗΣΑΤΕ|ΗΣΕΙ|ΗΣΕΣ|ΗΣΟΥΝ|ΗΣΩ|Ο|ΟΙ|ΟΜΑΙ|ΟΜΑΣΤΑΝ|ΟΜΟΥΝ|ΟΜΟΥΝΑ|ΟΝΤΑΙ|ΟΝΤΑΝ|ΟΝΤΟΥΣΑΝ|ΟΣ|ΟΣΑΣΤΑΝ|ΟΣΑΣΤΕ|ΟΣΟΥΝ|ΟΣΟΥΝΑ|ΟΤΑΝ|ΟΥ|ΟΥΜΑΙ|ΟΥΜΑΣΤΕ|ΟΥΝ|ΟΥΝΤΑΙ|ΟΥΝΤΑΝ|ΟΥΣ|ΟΥΣΑΝ|ΟΥΣΑΤΕ|Υ||ΥΑ|ΥΣ|Ω|ΩΝ|ΟΙΣ)$/.exec(s))&&(s=o[1]),null!=(o=/^(.+?)(ΕΣΤΕΡ|ΕΣΤΑΤ|ΟΤΕΡ|ΟΤΑΤ|ΥΤΕΡ|ΥΤΑΤ|ΩΤΕΡ|ΩΤΑΤ)$/.exec(s))&&(/^(ΕΞ|ΕΣ|ΑΝ|ΚΑΤ|Κ|ΠΡ)$/.test(o[1])||(s=o[1]),/^(ΚΑ|Μ|ΕΛΕ|ΛΕ|ΔΕ)$/.test(o[1])&&(s+="ΥΤ")),s}var l={"ΦΑΓΙΑ":"ΦΑ","ΦΑΓΙΟΥ":"ΦΑ","ΦΑΓΙΩΝ":"ΦΑ","ΣΚΑΓΙΑ":"ΣΚΑ","ΣΚΑΓΙΟΥ":"ΣΚΑ","ΣΚΑΓΙΩΝ":"ΣΚΑ","ΣΟΓΙΟΥ":"ΣΟ","ΣΟΓΙΑ":"ΣΟ","ΣΟΓΙΩΝ":"ΣΟ","ΤΑΤΟΓΙΑ":"ΤΑΤΟ","ΤΑΤΟΓΙΟΥ":"ΤΑΤΟ","ΤΑΤΟΓΙΩΝ":"ΤΑΤΟ","ΚΡΕΑΣ":"ΚΡΕ","ΚΡΕΑΤΟΣ":"ΚΡΕ","ΚΡΕΑΤΑ":"ΚΡΕ","ΚΡΕΑΤΩΝ":"ΚΡΕ","ΠΕΡΑΣ":"ΠΕΡ","ΠΕΡΑΤΟΣ":"ΠΕΡ","ΠΕΡΑΤΑ":"ΠΕΡ","ΠΕΡΑΤΩΝ":"ΠΕΡ","ΤΕΡΑΣ":"ΤΕΡ","ΤΕΡΑΤΟΣ":"ΤΕΡ","ΤΕΡΑΤΑ":"ΤΕΡ","ΤΕΡΑΤΩΝ":"ΤΕΡ","ΦΩΣ":"ΦΩ","ΦΩΤΟΣ":"ΦΩ","ΦΩΤΑ":"ΦΩ","ΦΩΤΩΝ":"ΦΩ","ΚΑΘΕΣΤΩΣ":"ΚΑΘΕΣΤ","ΚΑΘΕΣΤΩΤΟΣ":"ΚΑΘΕΣΤ","ΚΑΘΕΣΤΩΤΑ":"ΚΑΘΕΣΤ","ΚΑΘΕΣΤΩΤΩΝ":"ΚΑΘΕΣΤ","ΓΕΓΟΝΟΣ":"ΓΕΓΟΝ","ΓΕΓΟΝΟΤΟΣ":"ΓΕΓΟΝ","ΓΕΓΟΝΟΤΑ":"ΓΕΓΟΝ","ΓΕΓΟΝΟΤΩΝ":"ΓΕΓΟΝ","ΕΥΑ":"ΕΥ"},i=["ΑΚΡΙΒΩΣ","ΑΛΑ","ΑΛΛΑ","ΑΛΛΙΩΣ","ΑΛΛΟΤΕ","ΑΜΑ","ΑΝΩ","ΑΝΑ","ΑΝΑΜΕΣΑ","ΑΝΑΜΕΤΑΞΥ","ΑΝΕΥ","ΑΝΤΙ","ΑΝΤΙΠΕΡΑ","ΑΝΤΙΟ","ΑΞΑΦΝΑ","ΑΠΟ","ΑΠΟΨΕ","ΑΡΑ","ΑΡΑΓΕ","ΑΥΡΙΟ","ΑΦΟΙ","ΑΦΟΥ","ΑΦΟΤΟΥ","ΒΡΕ","ΓΕΙΑ","ΓΙΑ","ΓΙΑΤΙ","ΓΡΑΜΜΑ","ΔΕΗ","ΔΕΝ","ΔΗΛΑΔΗ","ΔΙΧΩΣ","ΔΥΟ","ΕΑΝ","ΕΓΩ","ΕΔΩ","ΕΔΑ","ΕΙΘΕ","ΕΙΜΑΙ","ΕΙΜΑΣΤΕ","ΕΙΣΑΙ","ΕΙΣΑΣΤΕ","ΕΙΝΑΙ","ΕΙΣΤΕ","ΕΙΤΕ","ΕΚΕΙ","ΕΚΟ","ΕΛΑ","ΕΜΑΣ","ΕΜΕΙΣ","ΕΝΤΕΛΩΣ","ΕΝΤΟΣ","ΕΝΤΩΜΕΤΑΞΥ","ΕΝΩ","ΕΞΙ","ΕΞΙΣΟΥ","ΕΞΗΣ","ΕΞΩ","ΕΟΚ","ΕΠΑΝΩ","ΕΠΕΙΔΗ","ΕΠΕΙΤΑ","ΕΠΙ","ΕΠΙΣΗΣ","ΕΠΟΜΕΝΩΣ","ΕΠΤΑ","ΕΣΑΣ","ΕΣΕΙΣ","ΕΣΤΩ","ΕΣΥ","ΕΣΩ","ΕΤΣΙ","ΕΥΓΕ","ΕΦΕ","ΕΦΕΞΗΣ","ΕΧΤΕΣ","ΕΩΣ","ΗΔΗ","ΗΜΙ","ΗΠΑ","ΗΤΟΙ","ΘΕΣ","ΙΔΙΩΣ","ΙΔΗ","ΙΚΑ","ΙΣΩΣ","ΚΑΘΕ","ΚΑΘΕΤΙ","ΚΑΘΟΛΟΥ","ΚΑΘΩΣ","ΚΑΙ","ΚΑΝ","ΚΑΠΟΤΕ","ΚΑΠΟΥ","ΚΑΤΑ","ΚΑΤΙ","ΚΑΤΟΠΙΝ","ΚΑΤΩ","ΚΕΙ","ΚΙΧ","ΚΚΕ","ΚΟΛΑΝ","ΚΥΡΙΩΣ","ΚΩΣ","ΜΑΚΑΡΙ","ΜΑΛΙΣΤΑ","ΜΑΛΛΟΝ","ΜΑΙ","ΜΑΟ","ΜΑΟΥΣ","ΜΑΣ","ΜΕΘΑΥΡΙΟ","ΜΕΣ","ΜΕΣΑ","ΜΕΤΑ","ΜΕΤΑΞΥ","ΜΕΧΡΙ","ΜΗΔΕ","ΜΗΝ","ΜΗΠΩΣ","ΜΗΤΕ","ΜΙΑ","ΜΙΑΣ","ΜΙΣ","ΜΜΕ","ΜΟΛΟΝΟΤΙ","ΜΟΥ","ΜΠΑ","ΜΠΑΣ","ΜΠΟΥΦΑΝ","ΜΠΡΟΣ","ΝΑΙ","ΝΕΣ","ΝΤΑ","ΝΤΕ","ΞΑΝΑ","ΟΗΕ","ΟΚΤΩ","ΟΜΩΣ","ΟΝΕ","ΟΠΑ","ΟΠΟΥ","ΟΠΩΣ","ΟΣΟ","ΟΤΑΝ","ΟΤΕ","ΟΤΙ","ΟΥΤΕ","ΟΧΙ","ΠΑΛΙ","ΠΑΝ","ΠΑΝΟ","ΠΑΝΤΟΤΕ","ΠΑΝΤΟΥ","ΠΑΝΤΩΣ","ΠΑΝΩ","ΠΑΡΑ","ΠΕΡΑ","ΠΕΡΙ","ΠΕΡΙΠΟΥ","ΠΙΑ","ΠΙΟ","ΠΙΣΩ","ΠΛΑΙ","ΠΛΕΟΝ","ΠΛΗΝ","ΠΟΤΕ","ΠΟΥ","ΠΡΟ","ΠΡΟΣ","ΠΡΟΧΤΕΣ","ΠΡΟΧΘΕΣ","ΡΟΔΙ","ΠΩΣ","ΣΑΙ","ΣΑΣ","ΣΑΝ","ΣΕΙΣ","ΣΙΑ","ΣΚΙ","ΣΟΙ","ΣΟΥ","ΣΡΙ","ΣΥΝ","ΣΥΝΑΜΑ","ΣΧΕΔΟΝ","ΤΑΔΕ","ΤΑΞΙ","ΤΑΧΑ","ΤΕΙ","ΤΗΝ","ΤΗΣ","ΤΙΠΟΤΑ","ΤΙΠΟΤΕ","ΤΙΣ","ΤΟΝ","ΤΟΤΕ","ΤΟΥ","ΤΟΥΣ","ΤΣΑ","ΤΣΕ","ΤΣΙ","ΤΣΟΥ","ΤΩΝ","ΥΠΟ","ΥΠΟΨΗ","ΥΠΟΨΙΝ","ΥΣΤΕΡΑ","ΦΕΤΟΣ","ΦΙΣ","ΦΠΑ","ΧΑΦ","ΧΘΕΣ","ΧΤΕΣ","ΧΩΡΙΣ","ΩΣ","ΩΣΑΝ","ΩΣΟΤΟΥ","ΩΣΠΟΥ","ΩΣΤΕ","ΩΣΤΟΣΟ"],s=new RegExp("^[ΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩ]+$");return function(e){return"function"==typeof e.update?e.update(function(e){return n(e.toUpperCase()).toLowerCase()}):n(e.toUpperCase()).toLowerCase()}}(),e.Pipeline.registerFunction(e.el.stemmer,"stemmer-el"),e.el.stopWordFilter=e.generateStopWordFilter("αλλα αν αντι απο αυτα αυτεσ αυτη αυτο αυτοι αυτοσ αυτουσ αυτων για δε δεν εαν ειμαι ειμαστε ειναι εισαι ειστε εκεινα εκεινεσ εκεινη εκεινο εκεινοι εκεινοσ εκεινουσ εκεινων ενω επι η θα ισωσ κ και κατα κι μα με μετα μη μην να ο οι ομωσ οπωσ οσο οτι παρα ποια ποιεσ ποιο ποιοι ποιοσ ποιουσ ποιων που προσ πωσ σε στη στην στο στον τα την τησ το τον τοτε του των ωσ".split(" ")),e.Pipeline.registerFunction(e.el.stopWordFilter,"stopWordFilter-el"),e.el.normilizer=function(){var e={"Ά":"Α","ά":"α","Έ":"Ε","έ":"ε","Ή":"Η","ή":"η","Ί":"Ι","ί":"ι","Ό":"Ο","ο":"ο","Ύ":"Υ","ύ":"υ","Ώ":"Ω","ώ":"ω","Ϊ":"Ι","ϊ":"ι","Ϋ":"Υ","ϋ":"υ","ΐ":"ι","ΰ":"υ"};return function(t){if("function"==typeof t.update)return t.update(function(t){for(var r="",n=0;n<t.length;n++)r+=e[t.charAt(n)]||t.charAt(n);return r});for(var r="",n=0;n<t.length;n++)r+=e[t.charAt(n)]||t.charAt(n);return r}}(),e.Pipeline.registerFunction(e.el.normilizer,"normilizer-el")}});
\ No newline at end of file
diff --git a/assets/javascripts/lunr/min/lunr.he.min.js b/assets/javascripts/lunr/min/lunr.he.min.js
new file mode 100644
index 000000000..b863d3eae
--- /dev/null
+++ b/assets/javascripts/lunr/min/lunr.he.min.js
@@ -0,0 +1 @@
+!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.he=function(){this.pipeline.reset(),this.pipeline.add(e.he.trimmer,e.he.stopWordFilter,e.he.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.he.stemmer))},e.he.wordCharacters="֑-״א-תa-zA-Zａ-ｚＡ-Ｚ0-9０-９",e.he.trimmer=e.trimmerSupport.generateTrimmer(e.he.wordCharacters),e.Pipeline.registerFunction(e.he.trimmer,"trimmer-he"),e.he.stemmer=function(){var e=this;return e.result=!1,e.preRemoved=!1,e.sufRemoved=!1,e.pre={pre1:"ה ו י ת",pre2:"ב כ ל מ ש כש",pre3:"הב הכ הל המ הש בש לכ",pre4:"וב וכ ול ומ וש",pre5:"מה שה כל",pre6:"מב מכ מל ממ מש",pre7:"בה בו בי בת כה כו כי כת לה לו לי לת",pre8:"ובה ובו ובי ובת וכה וכו וכי וכת ולה ולו ולי ולת"},e.suf={suf1:"ך כ ם ן נ",suf2:"ים ות וך וכ ום ון ונ הם הן יכ יך ינ ים",suf3:"תי תך תכ תם תן תנ",suf4:"ותי ותך ותכ ותם ותן ותנ",suf5:"נו כם כן הם הן",suf6:"ונו וכם וכן והם והן",suf7:"תכם תכן תנו תהם תהן",suf8:"הוא היא הם הן אני אתה את אנו אתם אתן",suf9:"ני נו כי כו כם כן תי תך תכ תם תן",suf10:"י ך כ ם ן נ ת"},e.patterns=JSON.parse('{"hebrewPatterns": [{"pt1": [{"c": "ה", "l": 0}]}, {"pt2": [{"c": "ו", "l": 0}]}, {"pt3": [{"c": "י", "l": 0}]}, {"pt4": [{"c": "ת", "l": 0}]}, {"pt5": [{"c": "מ", "l": 0}]}, {"pt6": [{"c": "ל", "l": 0}]}, {"pt7": [{"c": "ב", "l": 0}]}, {"pt8": [{"c": "כ", "l": 0}]}, {"pt9": [{"c": "ש", "l": 0}]}, {"pt10": [{"c": "כש", "l": 0}]}, {"pt11": [{"c": "בה", "l": 0}]}, {"pt12": [{"c": "וב", "l": 0}]}, {"pt13": [{"c": "וכ", "l": 0}]}, {"pt14": [{"c": "ול", "l": 0}]}, {"pt15": [{"c": "ומ", "l": 0}]}, {"pt16": [{"c": "וש", "l": 0}]}, {"pt17": [{"c": "הב", "l": 0}]}, {"pt18": [{"c": "הכ", "l": 0}]}, {"pt19": [{"c": "הל", "l": 0}]}, {"pt20": [{"c": "המ", "l": 0}]}, {"pt21": [{"c": "הש", "l": 0}]}, {"pt22": [{"c": "מה", "l": 0}]}, {"pt23": [{"c": "שה", "l": 0}]}, {"pt24": [{"c": "כל", "l": 0}]}]}'),e.execArray=["cleanWord","removeDiacritics","removeStopWords","normalizeHebrewCharacters"],e.stem=function(){var r=0;for(e.result=!1,e.preRemoved=!1,e.sufRemoved=!1;r<e.execArray.length&&1!=e.result;)e.result=e[e.execArray[r]](),r++},e.setCurrent=function(r){e.word=r},e.getCurrent=function(){return e.word},e.cleanWord=function(){return!!new RegExp("[^֑-״א-ת]").test("")},e.removeDiacritics=function(){var r=new RegExp("[ְ-ֿ]","g");return e.word=e.word.replace(r,""),!1},e.removeStopWords=function(){if("אבל או אולי אותו אותי אותך אותם אותן אותנו אז אחר אחרות אחרי אחריכן אחרים אחרת אי איזה איך אין איפה אל אלה אלו אם אנחנו אני אף אפשר את אתה אתכם אתכן אתם אתן באיזה באיזו בגלל בין בלבד בעבור בעזרת בכל בכן בלי במידה במקום שבו ברוב בשביל בשעה ש בתוך גם דרך הוא היא היה היי היכן היתה היתי הם הן הנה הסיבה שבגללה הרי ואילו ואת זאת זה זות יהיה יוכל יוכלו יותר מדי יכול יכולה יכולות יכולים יכל יכלה יכלו יש כאן כאשר כולם כולן כזה כי כיצד כך כל כלל כמו כן כפי כש לא לאו לאיזותך לאן לבין לה להיות להם להן לו לזה לזות לי לך לכם לכן למה למעלה למעלה מ למטה למטה מ למעט למקום שבו למרות לנו לעבר לעיכן לפיכך לפני מאד מאחורי מאיזו סיבה מאין מאיפה מבלי מבעד מדוע מה מהיכן מול מחוץ מי מידע מכאן מכל מכן מלבד מן מנין מסוגל מעט מעטים מעל מצד מקום בו מתחת מתי נגד נגר נו עד עז על עלי עליו עליה עליהם עליך עלינו עם עצמה עצמהם עצמהן עצמו עצמי עצמם עצמן עצמנו פה רק שוב של שלה שלהם שלהן שלו שלי שלך שלכה שלכם שלכן שלנו שם תהיה תחת".split(" ").indexOf(e.word)>=0)return!0},e.normalizeHebrewCharacters=function(){return e.word=e.word.replace("ך","כ"),e.word=e.word.replace("ם","מ"),e.word=e.word.replace("ן","נ"),e.word=e.word.replace("ף","פ"),e.word=e.word.replace("ץ","צ"),!1},function(r){return"function"==typeof r.update?r.update(function(r){return e.setCurrent(r),e.stem(),e.getCurrent()}):(e.setCurrent(r),e.stem(),e.getCurrent())}}(),e.Pipeline.registerFunction(e.he.stemmer,"stemmer-he"),e.he.stopWordFilter=e.generateStopWordFilter("אבל או אולי אותו אותי אותך אותם אותן אותנו אז אחר אחרות אחרי אחריכן אחרים אחרת אי איזה איך אין איפה אל אלה אלו אם אנחנו אני אף אפשר את אתה אתכם אתכן אתם אתן באיזה באיזו בגלל בין בלבד בעבור בעזרת בכל בכן בלי במידה במקום שבו ברוב בשביל בשעה ש בתוך גם דרך הוא היא היה היי היכן היתה היתי הם הן הנה הסיבה שבגללה הרי ואילו ואת זאת זה זות יהיה יוכל יוכלו יותר מדי יכול יכולה יכולות יכולים יכל יכלה יכלו יש כאן כאשר כולם כולן כזה כי כיצד כך כל כלל כמו כן כפי כש לא לאו לאיזותך לאן לבין לה להיות להם להן לו לזה לזות לי לך לכם לכן למה למעלה למעלה מ למטה למטה מ למעט למקום שבו למרות לנו לעבר לעיכן לפיכך לפני מאד מאחורי מאיזו סיבה מאין מאיפה מבלי מבעד מדוע מה מהיכן מול מחוץ מי מידע מכאן מכל מכן מלבד מן מנין מסוגל מעט מעטים מעל מצד מקום בו מתחת מתי נגד נגר נו עד עז על עלי עליו עליה עליהם עליך עלינו עם עצמה עצמהם עצמהן עצמו עצמי עצמם עצמן עצמנו פה רק שוב של שלה שלהם שלהן שלו שלי שלך שלכה שלכם שלכן שלנו שם תהיה תחת".split(" ")),e.Pipeline.registerFunction(e.he.stopWordFilter,"stopWordFilter-he")}});
\ No newline at end of file
diff --git a/assets/javascripts/lunr/min/lunr.hy.min.js b/assets/javascripts/lunr/min/lunr.hy.min.js
new file mode 100644
index 000000000..b37f79298
--- /dev/null
+++ b/assets/javascripts/lunr/min/lunr.hy.min.js
@@ -0,0 +1 @@
+!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.hy=function(){this.pipeline.reset(),this.pipeline.add(e.hy.trimmer,e.hy.stopWordFilter)},e.hy.wordCharacters="[A-Za-z԰-֏ff-ﭏ]",e.hy.trimmer=e.trimmerSupport.generateTrimmer(e.hy.wordCharacters),e.Pipeline.registerFunction(e.hy.trimmer,"trimmer-hy"),e.hy.stopWordFilter=e.generateStopWordFilter("դու և եք էիր էիք հետո նաև նրանք որը վրա է որ պիտի են այս մեջ ն իր ու ի այդ որոնք այն կամ էր մի ես համար այլ իսկ էին ենք հետ ին թ էինք մենք նրա նա դուք եմ էի ըստ որպես ում".split(" ")),e.Pipeline.registerFunction(e.hy.stopWordFilter,"stopWordFilter-hy"),e.hy.stemmer=function(){return function(e){return"function"==typeof e.update?e.update(function(e){return e}):e}}(),e.Pipeline.registerFunction(e.hy.stemmer,"stemmer-hy")}});
\ No newline at end of file
diff --git a/assets/javascripts/lunr/min/lunr.kn.min.js b/assets/javascripts/lunr/min/lunr.kn.min.js
new file mode 100644
index 000000000..1cef9befd
--- /dev/null
+++ b/assets/javascripts/lunr/min/lunr.kn.min.js
@@ -0,0 +1 @@
+!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.kn=function(){this.pipeline.reset(),this.pipeline.add(e.kn.trimmer,e.kn.stopWordFilter,e.kn.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.kn.stemmer))},e.kn.wordCharacters="ಀ-಄ಅ-ಔಕ-ಹಾ-ೌ಼-ಽೕ-ೖೝ-ೞೠ-ೡೢ-ೣ೤೥೦-೯ೱ-ೳ",e.kn.trimmer=e.trimmerSupport.generateTrimmer(e.kn.wordCharacters),e.Pipeline.registerFunction(e.kn.trimmer,"trimmer-kn"),e.kn.stopWordFilter=e.generateStopWordFilter("ಮತ್ತು ಈ ಒಂದು ರಲ್ಲಿ ಹಾಗೂ ಎಂದು ಅಥವಾ ಇದು ರ ಅವರು ಎಂಬ ಮೇಲೆ ಅವರ ತನ್ನ ಆದರೆ ತಮ್ಮ ನಂತರ ಮೂಲಕ ಹೆಚ್ಚು ನ ಆ ಕೆಲವು ಅನೇಕ ಎರಡು ಹಾಗು ಪ್ರಮುಖ ಇದನ್ನು ಇದರ ಸುಮಾರು ಅದರ ಅದು ಮೊದಲ ಬಗ್ಗೆ ನಲ್ಲಿ ರಂದು ಇತರ ಅತ್ಯಂತ ಹೆಚ್ಚಿನ ಸಹ ಸಾಮಾನ್ಯವಾಗಿ ನೇ ಹಲವಾರು ಹೊಸ ದಿ ಕಡಿಮೆ ಯಾವುದೇ ಹೊಂದಿದೆ ದೊಡ್ಡ ಅನ್ನು ಇವರು ಪ್ರಕಾರ ಇದೆ ಮಾತ್ರ ಕೂಡ ಇಲ್ಲಿ ಎಲ್ಲಾ ವಿವಿಧ ಅದನ್ನು ಹಲವು ರಿಂದ ಕೇವಲ ದ ದಕ್ಷಿಣ ಗೆ ಅವನ ಅತಿ ನೆಯ ಬಹಳ ಕೆಲಸ ಎಲ್ಲ ಪ್ರತಿ ಇತ್ಯಾದಿ ಇವು ಬೇರೆ ಹೀಗೆ ನಡುವೆ ಇದಕ್ಕೆ ಎಸ್ ಇವರ ಮೊದಲು ಶ್ರೀ ಮಾಡುವ ಇದರಲ್ಲಿ ರೀತಿಯ ಮಾಡಿದ ಕಾಲ ಅಲ್ಲಿ ಮಾಡಲು ಅದೇ ಈಗ ಅವು ಗಳು ಎ ಎಂಬುದು ಅವನು ಅಂದರೆ ಅವರಿಗೆ ಇರುವ ವಿಶೇಷ ಮುಂದೆ ಅವುಗಳ ಮುಂತಾದ ಮೂಲ ಬಿ ಮೀ ಒಂದೇ ಇನ್ನೂ ಹೆಚ್ಚಾಗಿ ಮಾಡಿ ಅವರನ್ನು ಇದೇ ಯ ರೀತಿಯಲ್ಲಿ ಜೊತೆ ಅದರಲ್ಲಿ ಮಾಡಿದರು ನಡೆದ ಆಗ ಮತ್ತೆ ಪೂರ್ವ ಆತ ಬಂದ ಯಾವ ಒಟ್ಟು ಇತರೆ ಹಿಂದೆ ಪ್ರಮಾಣದ ಗಳನ್ನು ಕುರಿತು ಯು ಆದ್ದರಿಂದ ಅಲ್ಲದೆ ನಗರದ ಮೇಲಿನ ಏಕೆಂದರೆ ರಷ್ಟು ಎಂಬುದನ್ನು ಬಾರಿ ಎಂದರೆ ಹಿಂದಿನ ಆದರೂ ಆದ ಸಂಬಂಧಿಸಿದ ಮತ್ತೊಂದು ಸಿ ಆತನ ".split(" ")),e.kn.stemmer=function(){return function(e){return"function"==typeof e.update?e.update(function(e){return e}):e}}();var r=e.wordcut;r.init(),e.kn.tokenizer=function(t){if(!arguments.length||null==t||void 0==t)return[];if(Array.isArray(t))return t.map(function(r){return isLunr2?new e.Token(r.toLowerCase()):r.toLowerCase()});var n=t.toString().toLowerCase().replace(/^\s+/,"");return r.cut(n).split("|")},e.Pipeline.registerFunction(e.kn.stemmer,"stemmer-kn"),e.Pipeline.registerFunction(e.kn.stopWordFilter,"stopWordFilter-kn")}});
\ No newline at end of file
diff --git a/assets/javascripts/lunr/min/lunr.ko.min.js b/assets/javascripts/lunr/min/lunr.ko.min.js
new file mode 100644
index 000000000..eaf9dabf7
--- /dev/null
+++ b/assets/javascripts/lunr/min/lunr.ko.min.js
@@ -0,0 +1 @@
+!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.ko=function(){this.pipeline.reset(),this.pipeline.add(e.ko.trimmer,e.ko.stopWordFilter)},e.ko.wordCharacters="[A-Za-z가-힣]",e.ko.trimmer=e.trimmerSupport.generateTrimmer(e.ko.wordCharacters),e.Pipeline.registerFunction(e.ko.trimmer,"trimmer-ko"),e.ko.stopWordFilter=e.generateStopWordFilter("아 휴 아이구 아이쿠 아이고 어 나 우리 저희 따라 의해 을 를 에 의 가 으로 로 에게 뿐이다 의거하여 근거하여 입각하여 기준으로 예하면 예를 들면 예를 들자면 저 소인 소생 저희 지말고 하지마 하지마라 다른 물론 또한 그리고 비길수 없다 해서는 안된다 뿐만 아니라 만이 아니다 만은 아니다 막론하고 관계없이 그치지 않다 그러나 그런데 하지만 든간에 논하지 않다 따지지 않다 설사 비록 더라도 아니면 만 못하다 하는 편이 낫다 불문하고 향하여 향해서 향하다 쪽으로 틈타 이용하여 타다 오르다 제외하고 이 외에 이 밖에 하여야 비로소 한다면 몰라도 외에도 이곳 여기 부터 기점으로 따라서 할 생각이다 하려고하다 이리하여 그리하여 그렇게 함으로써 하지만 일때 할때 앞에서 중에서 보는데서 으로써 로써 까지 해야한다 일것이다 반드시 할줄알다 할수있다 할수있어 임에 틀림없다 한다면 등 등등 제 겨우 단지 다만 할뿐 딩동 댕그 대해서 대하여 대하면 훨씬 얼마나 얼마만큼 얼마큼 남짓 여 얼마간 약간 다소 좀 조금 다수 몇 얼마 지만 하물며 또한 그러나 그렇지만 하지만 이외에도 대해 말하자면 뿐이다 다음에 반대로 반대로 말하자면 이와 반대로 바꾸어서 말하면 바꾸어서 한다면 만약 그렇지않으면 까악 툭 딱 삐걱거리다 보드득 비걱거리다 꽈당 응당 해야한다 에 가서 각 각각 여러분 각종 각자 제각기 하도록하다 와 과 그러므로 그래서 고로 한 까닭에 하기 때문에 거니와 이지만 대하여 관하여 관한 과연 실로 아니나다를가 생각한대로 진짜로 한적이있다 하곤하였다 하 하하 허허 아하 거바 와 오 왜 어째서 무엇때문에 어찌 하겠는가 무슨 어디 어느곳 더군다나 하물며 더욱이는 어느때 언제 야 이봐 어이 여보시오 흐흐 흥 휴 헉헉 헐떡헐떡 영차 여차 어기여차 끙끙 아야 앗 아야 콸콸 졸졸 좍좍 뚝뚝 주룩주룩 솨 우르르 그래도 또 그리고 바꾸어말하면 바꾸어말하자면 혹은 혹시 답다 및 그에 따르는 때가 되어 즉 지든지 설령 가령 하더라도 할지라도 일지라도 지든지 몇 거의 하마터면 인젠 이젠 된바에야 된이상 만큼\t어찌됏든 그위에 게다가 점에서 보아 비추어 보아 고려하면 하게될것이다 일것이다 비교적 좀 보다더 비하면 시키다 하게하다 할만하다 의해서 연이서 이어서 잇따라 뒤따라 뒤이어 결국 의지하여 기대여 통하여 자마자 더욱더 불구하고 얼마든지 마음대로 주저하지 않고 곧 즉시 바로 당장 하자마자 밖에 안된다 하면된다 그래 그렇지 요컨대 다시 말하자면 바꿔 말하면 즉 구체적으로 말하자면 시작하여 시초에 이상 허 헉 허걱 바와같이 해도좋다 해도된다 게다가 더구나 하물며 와르르 팍 퍽 펄렁 동안 이래 하고있었다 이었다 에서 로부터 까지 예하면 했어요 해요 함께 같이 더불어 마저 마저도 양자 모두 습니다 가까스로 하려고하다 즈음하여 다른 다른 방면으로 해봐요 습니까 했어요 말할것도 없고 무릎쓰고 개의치않고 하는것만 못하다 하는것이 낫다 매 매번 들 모 어느것 어느 로써 갖고말하자면 어디 어느쪽 어느것 어느해 어느 년도 라 해도 언젠가 어떤것 어느것 저기 저쪽 저것 그때 그럼 그러면 요만한걸 그래 그때 저것만큼 그저 이르기까지 할 줄 안다 할 힘이 있다 너 너희 당신 어찌 설마 차라리 할지언정 할지라도 할망정 할지언정 구토하다 게우다 토하다 메쓰겁다 옆사람 퉤 쳇 의거하여 근거하여 의해 따라 힘입어 그 다음 버금 두번째로 기타 첫번째로 나머지는 그중에서 견지에서 형식으로 쓰여 입장에서 위해서 단지 의해되다 하도록시키다 뿐만아니라 반대로 전후 전자 앞의것 잠시 잠깐 하면서 그렇지만 다음에 그러한즉 그런즉 남들 아무거나 어찌하든지 같다 비슷하다 예컨대 이럴정도로 어떻게 만약 만일 위에서 서술한바와같이 인 듯하다 하지 않는다면 만약에 무엇 무슨 어느 어떤 아래윗 조차 한데 그럼에도 불구하고 여전히 심지어 까지도 조차도 하지 않도록 않기 위하여 때 시각 무렵 시간 동안 어때 어떠한 하여금 네 예 우선 누구 누가 알겠는가 아무도 줄은모른다 줄은 몰랏다 하는 김에 겸사겸사 하는바 그런 까닭에 한 이유는 그러니 그러니까 때문에 그 너희 그들 너희들 타인 것 것들 너 위하여 공동으로 동시에 하기 위하여 어찌하여 무엇때문에 붕붕 윙윙 나 우리 엉엉 휘익 윙윙 오호 아하 어쨋든 만 못하다\t하기보다는 차라리 하는 편이 낫다 흐흐 놀라다 상대적으로 말하자면 마치 아니라면 쉿 그렇지 않으면 그렇지 않다면 안 그러면 아니었다면 하든지 아니면 이라면 좋아 알았어 하는것도 그만이다 어쩔수 없다 하나 일 일반적으로 일단 한켠으로는 오자마자 이렇게되면 이와같다면 전부 한마디 한항목 근거로 하기에 아울러 하지 않도록 않기 위해서 이르기까지 이 되다 로 인하여 까닭으로 이유만으로 이로 인하여 그래서 이 때문에 그러므로 그런 까닭에 알 수 있다 결론을 낼 수 있다 으로 인하여 있다 어떤것 관계가 있다 관련이 있다 연관되다 어떤것들 에 대해 이리하여 그리하여 여부 하기보다는 하느니 하면 할수록 운운 이러이러하다 하구나 하도다 다시말하면 다음으로 에 있다 에 달려 있다 우리 우리들 오히려 하기는한데 어떻게 어떻해 어찌됏어 어때 어째서 본대로 자 이 이쪽 여기 이것 이번 이렇게말하자면 이런 이러한 이와 같은 요만큼 요만한 것 얼마 안 되는 것 이만큼 이 정도의 이렇게 많은 것 이와 같다 이때 이렇구나 것과 같이 끼익 삐걱 따위 와 같은 사람들 부류의 사람들 왜냐하면 중의하나 오직 오로지 에 한하다 하기만 하면 도착하다 까지 미치다 도달하다 정도에 이르다 할 지경이다 결과에 이르다 관해서는 여러분 하고 있다 한 후 혼자 자기 자기집 자신 우에 종합한것과같이 총적으로 보면 총적으로 말하면 총적으로 대로 하다 으로서 참 그만이다 할 따름이다 쿵 탕탕 쾅쾅 둥둥 봐 봐라 아이야 아니 와아 응 아이 참나 년 월 일 령 영 일 이 삼 사 오 육 륙 칠 팔 구 이천육 이천칠 이천팔 이천구 하나 둘 셋 넷 다섯 여섯 일곱 여덟 아홉 령 영".split(" ")),e.Pipeline.registerFunction(e.ko.stopWordFilter,"stopWordFilter-ko"),e.ko.stemmer=function(){return function(e){return"function"==typeof e.update?e.update(function(e){return e}):e}}(),e.Pipeline.registerFunction(e.ko.stemmer,"stemmer-ko")}});
\ No newline at end of file
diff --git a/assets/javascripts/lunr/min/lunr.sa.min.js b/assets/javascripts/lunr/min/lunr.sa.min.js
new file mode 100644
index 000000000..50ee56420
--- /dev/null
+++ b/assets/javascripts/lunr/min/lunr.sa.min.js
@@ -0,0 +1 @@
+!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.sa=function(){this.pipeline.reset(),this.pipeline.add(e.sa.trimmer,e.sa.stopWordFilter,e.sa.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.sa.stemmer))},e.sa.wordCharacters="ऀ-ःऄ-एऐ-टठ-यर-िी-ॏॐ-य़ॠ-९॰-ॿ꣠-꣱ꣲ-ꣷ꣸-ꣻ꣼-ꣽꣾ-ꣿᆰ0-ᆰ9",e.sa.trimmer=e.trimmerSupport.generateTrimmer(e.sa.wordCharacters),e.Pipeline.registerFunction(e.sa.trimmer,"trimmer-sa"),e.sa.stopWordFilter=e.generateStopWordFilter('तथा अयम्‌ एकम्‌ इत्यस्मिन्‌ तथा तत्‌ वा अयम्‌ इत्यस्य ते आहूत उपरि तेषाम्‌  किन्तु तेषाम्‌ तदा इत्यनेन अधिकः इत्यस्य तत्‌ केचन बहवः द्वि तथा महत्वपूर्णः अयम्‌ अस्य  विषये अयं अस्ति तत्‌ प्रथमः विषये इत्युपरि इत्युपरि इतर अधिकतमः अधिकः अपि सामान्यतया ठ इतरेतर नूतनम्‌ द  न्यूनम्‌ कश्चित्‌ वा विशालः द  सः अस्ति तदनुसारम् तत्र अस्ति केवलम्‌ अपि अत्र सर्वे विविधाः तत्‌ बहवः यतः इदानीम्‌ द  दक्षिण इत्यस्मै तस्य उपरि नथ अतीव कार्यम्‌ सर्वे एकैकम्‌ इत्यादि। एते सन्ति  उत इत्थम्‌ मध्ये एतदर्थं . स कस्य प्रथमः श्री. करोति अस्मिन् प्रकारः निर्मिता कालः तत्र कर्तुं  समान अधुना ते सन्ति स एकः अस्ति सः अर्थात् तेषां कृते . स्थितम्  विशेषः अग्रिम तेषाम्‌ समान स्रोतः ख म समान इदानीमपि अधिकतया करोतु ते समान इत्यस्य वीथी सह यस्मिन्  कृतवान्‌ धृतः तदा पुनः पूर्वं सः आगतः किम्‌ कुल इतर पुरा  मात्रा स विषये उ अतएव अपि नगरस्य  उपरि यतः प्रतिशतं  कतरः कालः साधनानि भूत तथापि जात सम्बन्धि अन्यत्‌ ग अतः अस्माकं स्वकीयाः अस्माकं इदानीं अन्तः इत्यादयः भवन्तः इत्यादयः एते एताः तस्य अस्य इदम् एते तेषां तेषां तेषां तान् तेषां तेषां तेषां समानः सः एकः च तादृशाः बहवः अन्ये च वदन्ति यत् कियत् कस्मै  कस्मै  यस्मै  यस्मै  यस्मै  यस्मै न अतिनीचः किन्तु प्रथमं सम्पूर्णतया  ततः चिरकालानन्तरं पुस्तकं सम्पूर्णतया अन्तः  किन्तु अत्र वा इह इव श्रद्धाय अवशिष्यते  परन्तु अन्ये वर्गाः सन्ति ते सन्ति शक्नुवन्ति सर्वे मिलित्वा सर्वे एकत्र"'.split(" ")),e.sa.stemmer=function(){return function(e){return"function"==typeof e.update?e.update(function(e){return e}):e}}();var r=e.wordcut;r.init(),e.sa.tokenizer=function(t){if(!arguments.length||null==t||void 0==t)return[];if(Array.isArray(t))return t.map(function(r){return isLunr2?new e.Token(r.toLowerCase()):r.toLowerCase()});var i=t.toString().toLowerCase().replace(/^\s+/,"");return r.cut(i).split("|")},e.Pipeline.registerFunction(e.sa.stemmer,"stemmer-sa"),e.Pipeline.registerFunction(e.sa.stopWordFilter,"stopWordFilter-sa")}});
\ No newline at end of file
diff --git a/assets/javascripts/lunr/min/lunr.ta.min.js b/assets/javascripts/lunr/min/lunr.ta.min.js
new file mode 100644
index 000000000..a644bed22
--- /dev/null
+++ b/assets/javascripts/lunr/min/lunr.ta.min.js
@@ -0,0 +1 @@
+!function(e,t){"function"==typeof define&&define.amd?define(t):"object"==typeof exports?module.exports=t():t()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.ta=function(){this.pipeline.reset(),this.pipeline.add(e.ta.trimmer,e.ta.stopWordFilter,e.ta.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.ta.stemmer))},e.ta.wordCharacters="஀-உஊ-ஏஐ-ஙச-ட஠-னப-யர-ஹ஺-ிீ-௉ொ-௏ௐ-௙௚-௟௠-௩௪-௯௰-௹௺-௿a-zA-Zａ-ｚＡ-Ｚ0-9０-９",e.ta.trimmer=e.trimmerSupport.generateTrimmer(e.ta.wordCharacters),e.Pipeline.registerFunction(e.ta.trimmer,"trimmer-ta"),e.ta.stopWordFilter=e.generateStopWordFilter("அங்கு அங்கே அது அதை அந்த அவர் அவர்கள் அவள் அவன் அவை ஆக ஆகவே ஆகையால் ஆதலால் ஆதலினால் ஆனாலும் ஆனால் இங்கு இங்கே இது இதை இந்த இப்படி இவர் இவர்கள் இவள் இவன் இவை இவ்வளவு உனக்கு உனது உன் உன்னால் எங்கு எங்கே எது எதை எந்த எப்படி எவர் எவர்கள் எவள் எவன் எவை எவ்வளவு எனக்கு எனது எனவே என் என்ன என்னால் ஏது ஏன் தனது தன்னால் தானே தான் நாங்கள் நாம் நான் நீ நீங்கள்".split(" ")),e.ta.stemmer=function(){return function(e){return"function"==typeof e.update?e.update(function(e){return e}):e}}();var t=e.wordcut;t.init(),e.ta.tokenizer=function(r){if(!arguments.length||null==r||void 0==r)return[];if(Array.isArray(r))return r.map(function(t){return isLunr2?new e.Token(t.toLowerCase()):t.toLowerCase()});var i=r.toString().toLowerCase().replace(/^\s+/,"");return t.cut(i).split("|")},e.Pipeline.registerFunction(e.ta.stemmer,"stemmer-ta"),e.Pipeline.registerFunction(e.ta.stopWordFilter,"stopWordFilter-ta")}});
\ No newline at end of file
diff --git a/assets/javascripts/lunr/min/lunr.te.min.js b/assets/javascripts/lunr/min/lunr.te.min.js
new file mode 100644
index 000000000..9fa7a93b9
--- /dev/null
+++ b/assets/javascripts/lunr/min/lunr.te.min.js
@@ -0,0 +1 @@
+!function(e,t){"function"==typeof define&&define.amd?define(t):"object"==typeof exports?module.exports=t():t()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.te=function(){this.pipeline.reset(),this.pipeline.add(e.te.trimmer,e.te.stopWordFilter,e.te.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.te.stemmer))},e.te.wordCharacters="ఀ-ఄఅ-ఔక-హా-ౌౕ-ౖౘ-ౚౠ-ౡౢ-ౣ౦-౯౸-౿఼ఽ్ౝ౷౤౥",e.te.trimmer=e.trimmerSupport.generateTrimmer(e.te.wordCharacters),e.Pipeline.registerFunction(e.te.trimmer,"trimmer-te"),e.te.stopWordFilter=e.generateStopWordFilter("అందరూ అందుబాటులో అడగండి అడగడం అడ్డంగా అనుగుణంగా అనుమతించు అనుమతిస్తుంది అయితే ఇప్పటికే ఉన్నారు ఎక్కడైనా ఎప్పుడు ఎవరైనా ఎవరో ఏ ఏదైనా ఏమైనప్పటికి ఒక ఒకరు కనిపిస్తాయి కాదు కూడా గా గురించి చుట్టూ చేయగలిగింది తగిన తర్వాత దాదాపు దూరంగా నిజంగా పై ప్రకారం ప్రక్కన మధ్య మరియు మరొక మళ్ళీ మాత్రమే మెచ్చుకో వద్ద వెంట వేరుగా వ్యతిరేకంగా సంబంధం".split(" ")),e.te.stemmer=function(){return function(e){return"function"==typeof e.update?e.update(function(e){return e}):e}}();var t=e.wordcut;t.init(),e.te.tokenizer=function(r){if(!arguments.length||null==r||void 0==r)return[];if(Array.isArray(r))return r.map(function(t){return isLunr2?new e.Token(t.toLowerCase()):t.toLowerCase()});var i=r.toString().toLowerCase().replace(/^\s+/,"");return t.cut(i).split("|")},e.Pipeline.registerFunction(e.te.stemmer,"stemmer-te"),e.Pipeline.registerFunction(e.te.stopWordFilter,"stopWordFilter-te")}});
\ No newline at end of file
diff --git a/assets/javascripts/workers/search.b8dbb3d2.min.js b/assets/javascripts/workers/search.b8dbb3d2.min.js
new file mode 100644
index 000000000..c8a1ec8d8
--- /dev/null
+++ b/assets/javascripts/workers/search.b8dbb3d2.min.js
@@ -0,0 +1,42 @@
+"use strict";(()=>{var xe=Object.create;var U=Object.defineProperty,ve=Object.defineProperties,Se=Object.getOwnPropertyDescriptor,Te=Object.getOwnPropertyDescriptors,Qe=Object.getOwnPropertyNames,Y=Object.getOwnPropertySymbols,Ee=Object.getPrototypeOf,X=Object.prototype.hasOwnProperty,be=Object.prototype.propertyIsEnumerable;var Z=Math.pow,J=(t,e,r)=>e in t?U(t,e,{enumerable:!0,configurable:!0,writable:!0,value:r}):t[e]=r,A=(t,e)=>{for(var r in e||(e={}))X.call(e,r)&&J(t,r,e[r]);if(Y)for(var r of Y(e))be.call(e,r)&&J(t,r,e[r]);return t},G=(t,e)=>ve(t,Te(e));var Le=(t,e)=>()=>(e||t((e={exports:{}}).exports,e),e.exports);var we=(t,e,r,n)=>{if(e&&typeof e=="object"||typeof e=="function")for(let i of Qe(e))!X.call(t,i)&&i!==r&&U(t,i,{get:()=>e[i],enumerable:!(n=Se(e,i))||n.enumerable});return t};var Pe=(t,e,r)=>(r=t!=null?xe(Ee(t)):{},we(e||!t||!t.__esModule?U(r,"default",{value:t,enumerable:!0}):r,t));var B=(t,e,r)=>new Promise((n,i)=>{var s=u=>{try{a(r.next(u))}catch(c){i(c)}},o=u=>{try{a(r.throw(u))}catch(c){i(c)}},a=u=>u.done?n(u.value):Promise.resolve(u.value).then(s,o);a((r=r.apply(t,e)).next())});var te=Le((K,ee)=>{/**
+ * lunr - http://lunrjs.com - A bit like Solr, but much smaller and not as bright - 2.3.9
+ * Copyright (C) 2020 Oliver Nightingale
+ * @license MIT
+ */(function(){var t=function(e){var r=new t.Builder;return r.pipeline.add(t.trimmer,t.stopWordFilter,t.stemmer),r.searchPipeline.add(t.stemmer),e.call(r,r),r.build()};t.version="2.3.9";/*!
+ * lunr.utils
+ * Copyright (C) 2020 Oliver Nightingale
+ */t.utils={},t.utils.warn=function(e){return function(r){e.console&&console.warn&&console.warn(r)}}(this),t.utils.asString=function(e){return e==null?"":e.toString()},t.utils.clone=function(e){if(e==null)return e;for(var r=Object.create(null),n=Object.keys(e),i=0;i<n.length;i++){var s=n[i],o=e[s];if(Array.isArray(o)){r[s]=o.slice();continue}if(typeof o=="string"||typeof o=="number"||typeof o=="boolean"){r[s]=o;continue}throw new TypeError("clone is not deep and does not support nested objects")}return r},t.FieldRef=function(e,r,n){this.docRef=e,this.fieldName=r,this._stringValue=n},t.FieldRef.joiner="/",t.FieldRef.fromString=function(e){var r=e.indexOf(t.FieldRef.joiner);if(r===-1)throw"malformed field ref string";var n=e.slice(0,r),i=e.slice(r+1);return new t.FieldRef(i,n,e)},t.FieldRef.prototype.toString=function(){return this._stringValue==null&&(this._stringValue=this.fieldName+t.FieldRef.joiner+this.docRef),this._stringValue};/*!
+ * lunr.Set
+ * Copyright (C) 2020 Oliver Nightingale
+ */t.Set=function(e){if(this.elements=Object.create(null),e){this.length=e.length;for(var r=0;r<this.length;r++)this.elements[e[r]]=!0}else this.length=0},t.Set.complete={intersect:function(e){return e},union:function(){return this},contains:function(){return!0}},t.Set.empty={intersect:function(){return this},union:function(e){return e},contains:function(){return!1}},t.Set.prototype.contains=function(e){return!!this.elements[e]},t.Set.prototype.intersect=function(e){var r,n,i,s=[];if(e===t.Set.complete)return this;if(e===t.Set.empty)return e;this.length<e.length?(r=this,n=e):(r=e,n=this),i=Object.keys(r.elements);for(var o=0;o<i.length;o++){var a=i[o];a in n.elements&&s.push(a)}return new t.Set(s)},t.Set.prototype.union=function(e){return e===t.Set.complete?t.Set.complete:e===t.Set.empty?this:new t.Set(Object.keys(this.elements).concat(Object.keys(e.elements)))},t.idf=function(e,r){var n=0;for(var i in e)i!="_index"&&(n+=Object.keys(e[i]).length);var s=(r-n+.5)/(n+.5);return Math.log(1+Math.abs(s))},t.Token=function(e,r){this.str=e||"",this.metadata=r||{}},t.Token.prototype.toString=function(){return this.str},t.Token.prototype.update=function(e){return this.str=e(this.str,this.metadata),this},t.Token.prototype.clone=function(e){return e=e||function(r){return r},new t.Token(e(this.str,this.metadata),this.metadata)};/*!
+ * lunr.tokenizer
+ * Copyright (C) 2020 Oliver Nightingale
+ */t.tokenizer=function(e,r){if(e==null||e==null)return[];if(Array.isArray(e))return e.map(function(g){return new t.Token(t.utils.asString(g).toLowerCase(),t.utils.clone(r))});for(var n=e.toString().toLowerCase(),i=n.length,s=[],o=0,a=0;o<=i;o++){var u=n.charAt(o),c=o-a;if(u.match(t.tokenizer.separator)||o==i){if(c>0){var f=t.utils.clone(r)||{};f.position=[a,c],f.index=s.length,s.push(new t.Token(n.slice(a,o),f))}a=o+1}}return s},t.tokenizer.separator=/[\s\-]+/;/*!
+ * lunr.Pipeline
+ * Copyright (C) 2020 Oliver Nightingale
+ */t.Pipeline=function(){this._stack=[]},t.Pipeline.registeredFunctions=Object.create(null),t.Pipeline.registerFunction=function(e,r){r in this.registeredFunctions&&t.utils.warn("Overwriting existing registered function: "+r),e.label=r,t.Pipeline.registeredFunctions[e.label]=e},t.Pipeline.warnIfFunctionNotRegistered=function(e){var r=e.label&&e.label in this.registeredFunctions;r||t.utils.warn(`Function is not registered with pipeline. This may cause problems when serialising the index.
+`,e)},t.Pipeline.load=function(e){var r=new t.Pipeline;return e.forEach(function(n){var i=t.Pipeline.registeredFunctions[n];if(i)r.add(i);else throw new Error("Cannot load unregistered function: "+n)}),r},t.Pipeline.prototype.add=function(){var e=Array.prototype.slice.call(arguments);e.forEach(function(r){t.Pipeline.warnIfFunctionNotRegistered(r),this._stack.push(r)},this)},t.Pipeline.prototype.after=function(e,r){t.Pipeline.warnIfFunctionNotRegistered(r);var n=this._stack.indexOf(e);if(n==-1)throw new Error("Cannot find existingFn");n=n+1,this._stack.splice(n,0,r)},t.Pipeline.prototype.before=function(e,r){t.Pipeline.warnIfFunctionNotRegistered(r);var n=this._stack.indexOf(e);if(n==-1)throw new Error("Cannot find existingFn");this._stack.splice(n,0,r)},t.Pipeline.prototype.remove=function(e){var r=this._stack.indexOf(e);r!=-1&&this._stack.splice(r,1)},t.Pipeline.prototype.run=function(e){for(var r=this._stack.length,n=0;n<r;n++){for(var i=this._stack[n],s=[],o=0;o<e.length;o++){var a=i(e[o],o,e);if(!(a==null||a===""))if(Array.isArray(a))for(var u=0;u<a.length;u++)s.push(a[u]);else s.push(a)}e=s}return e},t.Pipeline.prototype.runString=function(e,r){var n=new t.Token(e,r);return this.run([n]).map(function(i){return i.toString()})},t.Pipeline.prototype.reset=function(){this._stack=[]},t.Pipeline.prototype.toJSON=function(){return this._stack.map(function(e){return t.Pipeline.warnIfFunctionNotRegistered(e),e.label})};/*!
+ * lunr.Vector
+ * Copyright (C) 2020 Oliver Nightingale
+ */t.Vector=function(e){this._magnitude=0,this.elements=e||[]},t.Vector.prototype.positionForIndex=function(e){if(this.elements.length==0)return 0;for(var r=0,n=this.elements.length/2,i=n-r,s=Math.floor(i/2),o=this.elements[s*2];i>1&&(o<e&&(r=s),o>e&&(n=s),o!=e);)i=n-r,s=r+Math.floor(i/2),o=this.elements[s*2];if(o==e||o>e)return s*2;if(o<e)return(s+1)*2},t.Vector.prototype.insert=function(e,r){this.upsert(e,r,function(){throw"duplicate index"})},t.Vector.prototype.upsert=function(e,r,n){this._magnitude=0;var i=this.positionForIndex(e);this.elements[i]==e?this.elements[i+1]=n(this.elements[i+1],r):this.elements.splice(i,0,e,r)},t.Vector.prototype.magnitude=function(){if(this._magnitude)return this._magnitude;for(var e=0,r=this.elements.length,n=1;n<r;n+=2){var i=this.elements[n];e+=i*i}return this._magnitude=Math.sqrt(e)},t.Vector.prototype.dot=function(e){for(var r=0,n=this.elements,i=e.elements,s=n.length,o=i.length,a=0,u=0,c=0,f=0;c<s&&f<o;)a=n[c],u=i[f],a<u?c+=2:a>u?f+=2:a==u&&(r+=n[c+1]*i[f+1],c+=2,f+=2);return r},t.Vector.prototype.similarity=function(e){return this.dot(e)/this.magnitude()||0},t.Vector.prototype.toArray=function(){for(var e=new Array(this.elements.length/2),r=1,n=0;r<this.elements.length;r+=2,n++)e[n]=this.elements[r];return e},t.Vector.prototype.toJSON=function(){return this.elements};/*!
+ * lunr.stemmer
+ * Copyright (C) 2020 Oliver Nightingale
+ * Includes code from - http://tartarus.org/~martin/PorterStemmer/js.txt
+ */t.stemmer=function(){var e={ational:"ate",tional:"tion",enci:"ence",anci:"ance",izer:"ize",bli:"ble",alli:"al",entli:"ent",eli:"e",ousli:"ous",ization:"ize",ation:"ate",ator:"ate",alism:"al",iveness:"ive",fulness:"ful",ousness:"ous",aliti:"al",iviti:"ive",biliti:"ble",logi:"log"},r={icate:"ic",ative:"",alize:"al",iciti:"ic",ical:"ic",ful:"",ness:""},n="[^aeiou]",i="[aeiouy]",s=n+"[^aeiouy]*",o=i+"[aeiou]*",a="^("+s+")?"+o+s,u="^("+s+")?"+o+s+"("+o+")?$",c="^("+s+")?"+o+s+o+s,f="^("+s+")?"+i,g=new RegExp(a),l=new RegExp(c),m=new RegExp(u),x=new RegExp(f),v=/^(.+?)(ss|i)es$/,d=/^(.+?)([^s])s$/,y=/^(.+?)eed$/,b=/^(.+?)(ed|ing)$/,E=/.$/,w=/(at|bl|iz)$/,R=new RegExp("([^aeiouylsz])\\1$"),j=new RegExp("^"+s+i+"[^aeiouwxy]$"),_=/^(.+?[^aeiou])y$/,D=/^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/,N=/^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/,C=/^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/,V=/^(.+?)(s|t)(ion)$/,P=/^(.+?)e$/,z=/ll$/,$=new RegExp("^"+s+i+"[^aeiouwxy]$"),M=function(h){var S,k,L,p,T,O,F;if(h.length<3)return h;if(L=h.substr(0,1),L=="y"&&(h=L.toUpperCase()+h.substr(1)),p=v,T=d,p.test(h)?h=h.replace(p,"$1$2"):T.test(h)&&(h=h.replace(T,"$1$2")),p=y,T=b,p.test(h)){var Q=p.exec(h);p=g,p.test(Q[1])&&(p=E,h=h.replace(p,""))}else if(T.test(h)){var Q=T.exec(h);S=Q[1],T=x,T.test(S)&&(h=S,T=w,O=R,F=j,T.test(h)?h=h+"e":O.test(h)?(p=E,h=h.replace(p,"")):F.test(h)&&(h=h+"e"))}if(p=_,p.test(h)){var Q=p.exec(h);S=Q[1],h=S+"i"}if(p=D,p.test(h)){var Q=p.exec(h);S=Q[1],k=Q[2],p=g,p.test(S)&&(h=S+e[k])}if(p=N,p.test(h)){var Q=p.exec(h);S=Q[1],k=Q[2],p=g,p.test(S)&&(h=S+r[k])}if(p=C,T=V,p.test(h)){var Q=p.exec(h);S=Q[1],p=l,p.test(S)&&(h=S)}else if(T.test(h)){var Q=T.exec(h);S=Q[1]+Q[2],T=l,T.test(S)&&(h=S)}if(p=P,p.test(h)){var Q=p.exec(h);S=Q[1],p=l,T=m,O=$,(p.test(S)||T.test(S)&&!O.test(S))&&(h=S)}return p=z,T=l,p.test(h)&&T.test(h)&&(p=E,h=h.replace(p,"")),L=="y"&&(h=L.toLowerCase()+h.substr(1)),h};return function(I){return I.update(M)}}(),t.Pipeline.registerFunction(t.stemmer,"stemmer");/*!
+ * lunr.stopWordFilter
+ * Copyright (C) 2020 Oliver Nightingale
+ */t.generateStopWordFilter=function(e){var r=e.reduce(function(n,i){return n[i]=i,n},{});return function(n){if(n&&r[n.toString()]!==n.toString())return n}},t.stopWordFilter=t.generateStopWordFilter(["a","able","about","across","after","all","almost","also","am","among","an","and","any","are","as","at","be","because","been","but","by","can","cannot","could","dear","did","do","does","either","else","ever","every","for","from","get","got","had","has","have","he","her","hers","him","his","how","however","i","if","in","into","is","it","its","just","least","let","like","likely","may","me","might","most","must","my","neither","no","nor","not","of","off","often","on","only","or","other","our","own","rather","said","say","says","she","should","since","so","some","than","that","the","their","them","then","there","these","they","this","tis","to","too","twas","us","wants","was","we","were","what","when","where","which","while","who","whom","why","will","with","would","yet","you","your"]),t.Pipeline.registerFunction(t.stopWordFilter,"stopWordFilter");/*!
+ * lunr.trimmer
+ * Copyright (C) 2020 Oliver Nightingale
+ */t.trimmer=function(e){return e.update(function(r){return r.replace(/^\W+/,"").replace(/\W+$/,"")})},t.Pipeline.registerFunction(t.trimmer,"trimmer");/*!
+ * lunr.TokenSet
+ * Copyright (C) 2020 Oliver Nightingale
+ */t.TokenSet=function(){this.final=!1,this.edges={},this.id=t.TokenSet._nextId,t.TokenSet._nextId+=1},t.TokenSet._nextId=1,t.TokenSet.fromArray=function(e){for(var r=new t.TokenSet.Builder,n=0,i=e.length;n<i;n++)r.insert(e[n]);return r.finish(),r.root},t.TokenSet.fromClause=function(e){return"editDistance"in e?t.TokenSet.fromFuzzyString(e.term,e.editDistance):t.TokenSet.fromString(e.term)},t.TokenSet.fromFuzzyString=function(e,r){for(var n=new t.TokenSet,i=[{node:n,editsRemaining:r,str:e}];i.length;){var s=i.pop();if(s.str.length>0){var o=s.str.charAt(0),a;o in s.node.edges?a=s.node.edges[o]:(a=new t.TokenSet,s.node.edges[o]=a),s.str.length==1&&(a.final=!0),i.push({node:a,editsRemaining:s.editsRemaining,str:s.str.slice(1)})}if(s.editsRemaining!=0){if("*"in s.node.edges)var u=s.node.edges["*"];else{var u=new t.TokenSet;s.node.edges["*"]=u}if(s.str.length==0&&(u.final=!0),i.push({node:u,editsRemaining:s.editsRemaining-1,str:s.str}),s.str.length>1&&i.push({node:s.node,editsRemaining:s.editsRemaining-1,str:s.str.slice(1)}),s.str.length==1&&(s.node.final=!0),s.str.length>=1){if("*"in s.node.edges)var c=s.node.edges["*"];else{var c=new t.TokenSet;s.node.edges["*"]=c}s.str.length==1&&(c.final=!0),i.push({node:c,editsRemaining:s.editsRemaining-1,str:s.str.slice(1)})}if(s.str.length>1){var f=s.str.charAt(0),g=s.str.charAt(1),l;g in s.node.edges?l=s.node.edges[g]:(l=new t.TokenSet,s.node.edges[g]=l),s.str.length==1&&(l.final=!0),i.push({node:l,editsRemaining:s.editsRemaining-1,str:f+s.str.slice(2)})}}}return n},t.TokenSet.fromString=function(e){for(var r=new t.TokenSet,n=r,i=0,s=e.length;i<s;i++){var o=e[i],a=i==s-1;if(o=="*")r.edges[o]=r,r.final=a;else{var u=new t.TokenSet;u.final=a,r.edges[o]=u,r=u}}return n},t.TokenSet.prototype.toArray=function(){for(var e=[],r=[{prefix:"",node:this}];r.length;){var n=r.pop(),i=Object.keys(n.node.edges),s=i.length;n.node.final&&(n.prefix.charAt(0),e.push(n.prefix));for(var o=0;o<s;o++){var a=i[o];r.push({prefix:n.prefix.concat(a),node:n.node.edges[a]})}}return e},t.TokenSet.prototype.toString=function(){if(this._str)return this._str;for(var e=this.final?"1":"0",r=Object.keys(this.edges).sort(),n=r.length,i=0;i<n;i++){var s=r[i],o=this.edges[s];e=e+s+o.id}return e},t.TokenSet.prototype.intersect=function(e){for(var r=new t.TokenSet,n=void 0,i=[{qNode:e,output:r,node:this}];i.length;){n=i.pop();for(var s=Object.keys(n.qNode.edges),o=s.length,a=Object.keys(n.node.edges),u=a.length,c=0;c<o;c++)for(var f=s[c],g=0;g<u;g++){var l=a[g];if(l==f||f=="*"){var m=n.node.edges[l],x=n.qNode.edges[f],v=m.final&&x.final,d=void 0;l in n.output.edges?(d=n.output.edges[l],d.final=d.final||v):(d=new t.TokenSet,d.final=v,n.output.edges[l]=d),i.push({qNode:x,output:d,node:m})}}}return r},t.TokenSet.Builder=function(){this.previousWord="",this.root=new t.TokenSet,this.uncheckedNodes=[],this.minimizedNodes={}},t.TokenSet.Builder.prototype.insert=function(e){var r,n=0;if(e<this.previousWord)throw new Error("Out of order word insertion");for(var i=0;i<e.length&&i<this.previousWord.length&&e[i]==this.previousWord[i];i++)n++;this.minimize(n),this.uncheckedNodes.length==0?r=this.root:r=this.uncheckedNodes[this.uncheckedNodes.length-1].child;for(var i=n;i<e.length;i++){var s=new t.TokenSet,o=e[i];r.edges[o]=s,this.uncheckedNodes.push({parent:r,char:o,child:s}),r=s}r.final=!0,this.previousWord=e},t.TokenSet.Builder.prototype.finish=function(){this.minimize(0)},t.TokenSet.Builder.prototype.minimize=function(e){for(var r=this.uncheckedNodes.length-1;r>=e;r--){var n=this.uncheckedNodes[r],i=n.child.toString();i in this.minimizedNodes?n.parent.edges[n.char]=this.minimizedNodes[i]:(n.child._str=i,this.minimizedNodes[i]=n.child),this.uncheckedNodes.pop()}};/*!
+ * lunr.Index
+ * Copyright (C) 2020 Oliver Nightingale
+ */t.Index=function(e){this.invertedIndex=e.invertedIndex,this.fieldVectors=e.fieldVectors,this.tokenSet=e.tokenSet,this.fields=e.fields,this.pipeline=e.pipeline},t.Index.prototype.search=function(e){return this.query(function(r){var n=new t.QueryParser(e,r);n.parse()})},t.Index.prototype.query=function(e){for(var r=new t.Query(this.fields),n=Object.create(null),i=Object.create(null),s=Object.create(null),o=Object.create(null),a=Object.create(null),u=0;u<this.fields.length;u++)i[this.fields[u]]=new t.Vector;e.call(r,r);for(var u=0;u<r.clauses.length;u++){var c=r.clauses[u],f=null,g=t.Set.empty;c.usePipeline?f=this.pipeline.runString(c.term,{fields:c.fields}):f=[c.term];for(var l=0;l<f.length;l++){var m=f[l];c.term=m;var x=t.TokenSet.fromClause(c),v=this.tokenSet.intersect(x).toArray();if(v.length===0&&c.presence===t.Query.presence.REQUIRED){for(var d=0;d<c.fields.length;d++){var y=c.fields[d];o[y]=t.Set.empty}break}for(var b=0;b<v.length;b++)for(var E=v[b],w=this.invertedIndex[E],R=w._index,d=0;d<c.fields.length;d++){var y=c.fields[d],j=w[y],_=Object.keys(j),D=E+"/"+y,N=new t.Set(_);if(c.presence==t.Query.presence.REQUIRED&&(g=g.union(N),o[y]===void 0&&(o[y]=t.Set.complete)),c.presence==t.Query.presence.PROHIBITED){a[y]===void 0&&(a[y]=t.Set.empty),a[y]=a[y].union(N);continue}if(i[y].upsert(R,c.boost,function(ye,me){return ye+me}),!s[D]){for(var C=0;C<_.length;C++){var V=_[C],P=new t.FieldRef(V,y),z=j[V],$;($=n[P])===void 0?n[P]=new t.MatchData(E,y,z):$.add(E,y,z)}s[D]=!0}}}if(c.presence===t.Query.presence.REQUIRED)for(var d=0;d<c.fields.length;d++){var y=c.fields[d];o[y]=o[y].intersect(g)}}for(var M=t.Set.complete,I=t.Set.empty,u=0;u<this.fields.length;u++){var y=this.fields[u];o[y]&&(M=M.intersect(o[y])),a[y]&&(I=I.union(a[y]))}var h=Object.keys(n),S=[],k=Object.create(null);if(r.isNegated()){h=Object.keys(this.fieldVectors);for(var u=0;u<h.length;u++){var P=h[u],L=t.FieldRef.fromString(P);n[P]=new t.MatchData}}for(var u=0;u<h.length;u++){var L=t.FieldRef.fromString(h[u]),p=L.docRef;if(M.contains(p)&&!I.contains(p)){var T=this.fieldVectors[L],O=i[L.fieldName].similarity(T),F;if((F=k[p])!==void 0)F.score+=O,F.matchData.combine(n[L]);else{var Q={ref:p,score:O,matchData:n[L]};k[p]=Q,S.push(Q)}}}return S.sort(function(pe,ge){return ge.score-pe.score})},t.Index.prototype.toJSON=function(){var e=Object.keys(this.invertedIndex).sort().map(function(n){return[n,this.invertedIndex[n]]},this),r=Object.keys(this.fieldVectors).map(function(n){return[n,this.fieldVectors[n].toJSON()]},this);return{version:t.version,fields:this.fields,fieldVectors:r,invertedIndex:e,pipeline:this.pipeline.toJSON()}},t.Index.load=function(e){var r={},n={},i=e.fieldVectors,s=Object.create(null),o=e.invertedIndex,a=new t.TokenSet.Builder,u=t.Pipeline.load(e.pipeline);e.version!=t.version&&t.utils.warn("Version mismatch when loading serialised index. Current version of lunr '"+t.version+"' does not match serialized index '"+e.version+"'");for(var c=0;c<i.length;c++){var f=i[c],g=f[0],l=f[1];n[g]=new t.Vector(l)}for(var c=0;c<o.length;c++){var f=o[c],m=f[0],x=f[1];a.insert(m),s[m]=x}return a.finish(),r.fields=e.fields,r.fieldVectors=n,r.invertedIndex=s,r.tokenSet=a.root,r.pipeline=u,new t.Index(r)};/*!
+ * lunr.Builder
+ * Copyright (C) 2020 Oliver Nightingale
+ */t.Builder=function(){this._ref="id",this._fields=Object.create(null),this._documents=Object.create(null),this.invertedIndex=Object.create(null),this.fieldTermFrequencies={},this.fieldLengths={},this.tokenizer=t.tokenizer,this.pipeline=new t.Pipeline,this.searchPipeline=new t.Pipeline,this.documentCount=0,this._b=.75,this._k1=1.2,this.termIndex=0,this.metadataWhitelist=[]},t.Builder.prototype.ref=function(e){this._ref=e},t.Builder.prototype.field=function(e,r){if(/\//.test(e))throw new RangeError("Field '"+e+"' contains illegal character '/'");this._fields[e]=r||{}},t.Builder.prototype.b=function(e){e<0?this._b=0:e>1?this._b=1:this._b=e},t.Builder.prototype.k1=function(e){this._k1=e},t.Builder.prototype.add=function(e,r){var n=e[this._ref],i=Object.keys(this._fields);this._documents[n]=r||{},this.documentCount+=1;for(var s=0;s<i.length;s++){var o=i[s],a=this._fields[o].extractor,u=a?a(e):e[o],c=this.tokenizer(u,{fields:[o]}),f=this.pipeline.run(c),g=new t.FieldRef(n,o),l=Object.create(null);this.fieldTermFrequencies[g]=l,this.fieldLengths[g]=0,this.fieldLengths[g]+=f.length;for(var m=0;m<f.length;m++){var x=f[m];if(l[x]==null&&(l[x]=0),l[x]+=1,this.invertedIndex[x]==null){var v=Object.create(null);v._index=this.termIndex,this.termIndex+=1;for(var d=0;d<i.length;d++)v[i[d]]=Object.create(null);this.invertedIndex[x]=v}this.invertedIndex[x][o][n]==null&&(this.invertedIndex[x][o][n]=Object.create(null));for(var y=0;y<this.metadataWhitelist.length;y++){var b=this.metadataWhitelist[y],E=x.metadata[b];this.invertedIndex[x][o][n][b]==null&&(this.invertedIndex[x][o][n][b]=[]),this.invertedIndex[x][o][n][b].push(E)}}}},t.Builder.prototype.calculateAverageFieldLengths=function(){for(var e=Object.keys(this.fieldLengths),r=e.length,n={},i={},s=0;s<r;s++){var o=t.FieldRef.fromString(e[s]),a=o.fieldName;i[a]||(i[a]=0),i[a]+=1,n[a]||(n[a]=0),n[a]+=this.fieldLengths[o]}for(var u=Object.keys(this._fields),s=0;s<u.length;s++){var c=u[s];n[c]=n[c]/i[c]}this.averageFieldLength=n},t.Builder.prototype.createFieldVectors=function(){for(var e={},r=Object.keys(this.fieldTermFrequencies),n=r.length,i=Object.create(null),s=0;s<n;s++){for(var o=t.FieldRef.fromString(r[s]),a=o.fieldName,u=this.fieldLengths[o],c=new t.Vector,f=this.fieldTermFrequencies[o],g=Object.keys(f),l=g.length,m=this._fields[a].boost||1,x=this._documents[o.docRef].boost||1,v=0;v<l;v++){var d=g[v],y=f[d],b=this.invertedIndex[d]._index,E,w,R;i[d]===void 0?(E=t.idf(this.invertedIndex[d],this.documentCount),i[d]=E):E=i[d],w=E*((this._k1+1)*y)/(this._k1*(1-this._b+this._b*(u/this.averageFieldLength[a]))+y),w*=m,w*=x,R=Math.round(w*1e3)/1e3,c.insert(b,R)}e[o]=c}this.fieldVectors=e},t.Builder.prototype.createTokenSet=function(){this.tokenSet=t.TokenSet.fromArray(Object.keys(this.invertedIndex).sort())},t.Builder.prototype.build=function(){return this.calculateAverageFieldLengths(),this.createFieldVectors(),this.createTokenSet(),new t.Index({invertedIndex:this.invertedIndex,fieldVectors:this.fieldVectors,tokenSet:this.tokenSet,fields:Object.keys(this._fields),pipeline:this.searchPipeline})},t.Builder.prototype.use=function(e){var r=Array.prototype.slice.call(arguments,1);r.unshift(this),e.apply(this,r)},t.MatchData=function(e,r,n){for(var i=Object.create(null),s=Object.keys(n||{}),o=0;o<s.length;o++){var a=s[o];i[a]=n[a].slice()}this.metadata=Object.create(null),e!==void 0&&(this.metadata[e]=Object.create(null),this.metadata[e][r]=i)},t.MatchData.prototype.combine=function(e){for(var r=Object.keys(e.metadata),n=0;n<r.length;n++){var i=r[n],s=Object.keys(e.metadata[i]);this.metadata[i]==null&&(this.metadata[i]=Object.create(null));for(var o=0;o<s.length;o++){var a=s[o],u=Object.keys(e.metadata[i][a]);this.metadata[i][a]==null&&(this.metadata[i][a]=Object.create(null));for(var c=0;c<u.length;c++){var f=u[c];this.metadata[i][a][f]==null?this.metadata[i][a][f]=e.metadata[i][a][f]:this.metadata[i][a][f]=this.metadata[i][a][f].concat(e.metadata[i][a][f])}}}},t.MatchData.prototype.add=function(e,r,n){if(!(e in this.metadata)){this.metadata[e]=Object.create(null),this.metadata[e][r]=n;return}if(!(r in this.metadata[e])){this.metadata[e][r]=n;return}for(var i=Object.keys(n),s=0;s<i.length;s++){var o=i[s];o in this.metadata[e][r]?this.metadata[e][r][o]=this.metadata[e][r][o].concat(n[o]):this.metadata[e][r][o]=n[o]}},t.Query=function(e){this.clauses=[],this.allFields=e},t.Query.wildcard=new String("*"),t.Query.wildcard.NONE=0,t.Query.wildcard.LEADING=1,t.Query.wildcard.TRAILING=2,t.Query.presence={OPTIONAL:1,REQUIRED:2,PROHIBITED:3},t.Query.prototype.clause=function(e){return"fields"in e||(e.fields=this.allFields),"boost"in e||(e.boost=1),"usePipeline"in e||(e.usePipeline=!0),"wildcard"in e||(e.wildcard=t.Query.wildcard.NONE),e.wildcard&t.Query.wildcard.LEADING&&e.term.charAt(0)!=t.Query.wildcard&&(e.term="*"+e.term),e.wildcard&t.Query.wildcard.TRAILING&&e.term.slice(-1)!=t.Query.wildcard&&(e.term=""+e.term+"*"),"presence"in e||(e.presence=t.Query.presence.OPTIONAL),this.clauses.push(e),this},t.Query.prototype.isNegated=function(){for(var e=0;e<this.clauses.length;e++)if(this.clauses[e].presence!=t.Query.presence.PROHIBITED)return!1;return!0},t.Query.prototype.term=function(e,r){if(Array.isArray(e))return e.forEach(function(i){this.term(i,t.utils.clone(r))},this),this;var n=r||{};return n.term=e.toString(),this.clause(n),this},t.QueryParseError=function(e,r,n){this.name="QueryParseError",this.message=e,this.start=r,this.end=n},t.QueryParseError.prototype=new Error,t.QueryLexer=function(e){this.lexemes=[],this.str=e,this.length=e.length,this.pos=0,this.start=0,this.escapeCharPositions=[]},t.QueryLexer.prototype.run=function(){for(var e=t.QueryLexer.lexText;e;)e=e(this)},t.QueryLexer.prototype.sliceString=function(){for(var e=[],r=this.start,n=this.pos,i=0;i<this.escapeCharPositions.length;i++)n=this.escapeCharPositions[i],e.push(this.str.slice(r,n)),r=n+1;return e.push(this.str.slice(r,this.pos)),this.escapeCharPositions.length=0,e.join("")},t.QueryLexer.prototype.emit=function(e){this.lexemes.push({type:e,str:this.sliceString(),start:this.start,end:this.pos}),this.start=this.pos},t.QueryLexer.prototype.escapeCharacter=function(){this.escapeCharPositions.push(this.pos-1),this.pos+=1},t.QueryLexer.prototype.next=function(){if(this.pos>=this.length)return t.QueryLexer.EOS;var e=this.str.charAt(this.pos);return this.pos+=1,e},t.QueryLexer.prototype.width=function(){return this.pos-this.start},t.QueryLexer.prototype.ignore=function(){this.start==this.pos&&(this.pos+=1),this.start=this.pos},t.QueryLexer.prototype.backup=function(){this.pos-=1},t.QueryLexer.prototype.acceptDigitRun=function(){var e,r;do e=this.next(),r=e.charCodeAt(0);while(r>47&&r<58);e!=t.QueryLexer.EOS&&this.backup()},t.QueryLexer.prototype.more=function(){return this.pos<this.length},t.QueryLexer.EOS="EOS",t.QueryLexer.FIELD="FIELD",t.QueryLexer.TERM="TERM",t.QueryLexer.EDIT_DISTANCE="EDIT_DISTANCE",t.QueryLexer.BOOST="BOOST",t.QueryLexer.PRESENCE="PRESENCE",t.QueryLexer.lexField=function(e){return e.backup(),e.emit(t.QueryLexer.FIELD),e.ignore(),t.QueryLexer.lexText},t.QueryLexer.lexTerm=function(e){if(e.width()>1&&(e.backup(),e.emit(t.QueryLexer.TERM)),e.ignore(),e.more())return t.QueryLexer.lexText},t.QueryLexer.lexEditDistance=function(e){return e.ignore(),e.acceptDigitRun(),e.emit(t.QueryLexer.EDIT_DISTANCE),t.QueryLexer.lexText},t.QueryLexer.lexBoost=function(e){return e.ignore(),e.acceptDigitRun(),e.emit(t.QueryLexer.BOOST),t.QueryLexer.lexText},t.QueryLexer.lexEOS=function(e){e.width()>0&&e.emit(t.QueryLexer.TERM)},t.QueryLexer.termSeparator=t.tokenizer.separator,t.QueryLexer.lexText=function(e){for(;;){var r=e.next();if(r==t.QueryLexer.EOS)return t.QueryLexer.lexEOS;if(r.charCodeAt(0)==92){e.escapeCharacter();continue}if(r==":")return t.QueryLexer.lexField;if(r=="~")return e.backup(),e.width()>0&&e.emit(t.QueryLexer.TERM),t.QueryLexer.lexEditDistance;if(r=="^")return e.backup(),e.width()>0&&e.emit(t.QueryLexer.TERM),t.QueryLexer.lexBoost;if(r=="+"&&e.width()===1||r=="-"&&e.width()===1)return e.emit(t.QueryLexer.PRESENCE),t.QueryLexer.lexText;if(r.match(t.QueryLexer.termSeparator))return t.QueryLexer.lexTerm}},t.QueryParser=function(e,r){this.lexer=new t.QueryLexer(e),this.query=r,this.currentClause={},this.lexemeIdx=0},t.QueryParser.prototype.parse=function(){this.lexer.run(),this.lexemes=this.lexer.lexemes;for(var e=t.QueryParser.parseClause;e;)e=e(this);return this.query},t.QueryParser.prototype.peekLexeme=function(){return this.lexemes[this.lexemeIdx]},t.QueryParser.prototype.consumeLexeme=function(){var e=this.peekLexeme();return this.lexemeIdx+=1,e},t.QueryParser.prototype.nextClause=function(){var e=this.currentClause;this.query.clause(e),this.currentClause={}},t.QueryParser.parseClause=function(e){var r=e.peekLexeme();if(r!=null)switch(r.type){case t.QueryLexer.PRESENCE:return t.QueryParser.parsePresence;case t.QueryLexer.FIELD:return t.QueryParser.parseField;case t.QueryLexer.TERM:return t.QueryParser.parseTerm;default:var n="expected either a field or a term, found "+r.type;throw r.str.length>=1&&(n+=" with value '"+r.str+"'"),new t.QueryParseError(n,r.start,r.end)}},t.QueryParser.parsePresence=function(e){var r=e.consumeLexeme();if(r!=null){switch(r.str){case"-":e.currentClause.presence=t.Query.presence.PROHIBITED;break;case"+":e.currentClause.presence=t.Query.presence.REQUIRED;break;default:var n="unrecognised presence operator'"+r.str+"'";throw new t.QueryParseError(n,r.start,r.end)}var i=e.peekLexeme();if(i==null){var n="expecting term or field, found nothing";throw new t.QueryParseError(n,r.start,r.end)}switch(i.type){case t.QueryLexer.FIELD:return t.QueryParser.parseField;case t.QueryLexer.TERM:return t.QueryParser.parseTerm;default:var n="expecting term or field, found '"+i.type+"'";throw new t.QueryParseError(n,i.start,i.end)}}},t.QueryParser.parseField=function(e){var r=e.consumeLexeme();if(r!=null){if(e.query.allFields.indexOf(r.str)==-1){var n=e.query.allFields.map(function(o){return"'"+o+"'"}).join(", "),i="unrecognised field '"+r.str+"', possible fields: "+n;throw new t.QueryParseError(i,r.start,r.end)}e.currentClause.fields=[r.str];var s=e.peekLexeme();if(s==null){var i="expecting term, found nothing";throw new t.QueryParseError(i,r.start,r.end)}switch(s.type){case t.QueryLexer.TERM:return t.QueryParser.parseTerm;default:var i="expecting term, found '"+s.type+"'";throw new t.QueryParseError(i,s.start,s.end)}}},t.QueryParser.parseTerm=function(e){var r=e.consumeLexeme();if(r!=null){e.currentClause.term=r.str.toLowerCase(),r.str.indexOf("*")!=-1&&(e.currentClause.usePipeline=!1);var n=e.peekLexeme();if(n==null){e.nextClause();return}switch(n.type){case t.QueryLexer.TERM:return e.nextClause(),t.QueryParser.parseTerm;case t.QueryLexer.FIELD:return e.nextClause(),t.QueryParser.parseField;case t.QueryLexer.EDIT_DISTANCE:return t.QueryParser.parseEditDistance;case t.QueryLexer.BOOST:return t.QueryParser.parseBoost;case t.QueryLexer.PRESENCE:return e.nextClause(),t.QueryParser.parsePresence;default:var i="Unexpected lexeme type '"+n.type+"'";throw new t.QueryParseError(i,n.start,n.end)}}},t.QueryParser.parseEditDistance=function(e){var r=e.consumeLexeme();if(r!=null){var n=parseInt(r.str,10);if(isNaN(n)){var i="edit distance must be numeric";throw new t.QueryParseError(i,r.start,r.end)}e.currentClause.editDistance=n;var s=e.peekLexeme();if(s==null){e.nextClause();return}switch(s.type){case t.QueryLexer.TERM:return e.nextClause(),t.QueryParser.parseTerm;case t.QueryLexer.FIELD:return e.nextClause(),t.QueryParser.parseField;case t.QueryLexer.EDIT_DISTANCE:return t.QueryParser.parseEditDistance;case t.QueryLexer.BOOST:return t.QueryParser.parseBoost;case t.QueryLexer.PRESENCE:return e.nextClause(),t.QueryParser.parsePresence;default:var i="Unexpected lexeme type '"+s.type+"'";throw new t.QueryParseError(i,s.start,s.end)}}},t.QueryParser.parseBoost=function(e){var r=e.consumeLexeme();if(r!=null){var n=parseInt(r.str,10);if(isNaN(n)){var i="boost must be numeric";throw new t.QueryParseError(i,r.start,r.end)}e.currentClause.boost=n;var s=e.peekLexeme();if(s==null){e.nextClause();return}switch(s.type){case t.QueryLexer.TERM:return e.nextClause(),t.QueryParser.parseTerm;case t.QueryLexer.FIELD:return e.nextClause(),t.QueryParser.parseField;case t.QueryLexer.EDIT_DISTANCE:return t.QueryParser.parseEditDistance;case t.QueryLexer.BOOST:return t.QueryParser.parseBoost;case t.QueryLexer.PRESENCE:return e.nextClause(),t.QueryParser.parsePresence;default:var i="Unexpected lexeme type '"+s.type+"'";throw new t.QueryParseError(i,s.start,s.end)}}},function(e,r){typeof define=="function"&&define.amd?define(r):typeof K=="object"?ee.exports=r():e.lunr=r()}(this,function(){return t})})()});var de=Pe(te());function re(t,e=document){let r=ke(t,e);if(typeof r=="undefined")throw new ReferenceError(`Missing element: expected "${t}" to be present`);return r}function ke(t,e=document){return e.querySelector(t)||void 0}Object.entries||(Object.entries=function(t){let e=[];for(let r of Object.keys(t))e.push([r,t[r]]);return e});Object.values||(Object.values=function(t){let e=[];for(let r of Object.keys(t))e.push(t[r]);return e});typeof Element!="undefined"&&(Element.prototype.scrollTo||(Element.prototype.scrollTo=function(t,e){typeof t=="object"?(this.scrollLeft=t.left,this.scrollTop=t.top):(this.scrollLeft=t,this.scrollTop=e)}),Element.prototype.replaceWith||(Element.prototype.replaceWith=function(...t){let e=this.parentNode;if(e){t.length===0&&e.removeChild(this);for(let r=t.length-1;r>=0;r--){let n=t[r];typeof n=="string"?n=document.createTextNode(n):n.parentNode&&n.parentNode.removeChild(n),r?e.insertBefore(this.previousSibling,n):e.replaceChild(n,this)}}}));function ne(t){let e=new Map;for(let r of t){let[n]=r.location.split("#"),i=e.get(n);typeof i=="undefined"?e.set(n,r):(e.set(r.location,r),r.parent=i)}return e}function W(t,e,r){var s;e=new RegExp(e,"g");let n,i=0;do{n=e.exec(t);let o=(s=n==null?void 0:n.index)!=null?s:t.length;if(i<o&&r(i,o),n){let[a]=n;i=n.index+a.length,a.length===0&&(e.lastIndex=n.index+1)}}while(n)}function ie(t,e){let r=0,n=0,i=0;for(let s=0;i<t.length;i++)t.charAt(i)==="<"&&i>n?e(r,1,n,n=i):t.charAt(i)===">"&&(t.charAt(n+1)==="/"?--s===0&&e(r++,2,n,i+1):t.charAt(i-1)!=="/"&&s++===0&&e(r,0,n,i+1),n=i+1);i>n&&e(r,1,n,i)}function se(t,e,r,n=!1){return q([t],e,r,n).pop()}function q(t,e,r,n=!1){let i=[0];for(let s=1;s<e.length;s++){let o=e[s-1],a=e[s],u=o[o.length-1]>>>2&1023,c=a[0]>>>12;i.push(+(u>c)+i[i.length-1])}return t.map((s,o)=>{let a=0,u=new Map;for(let f of r.sort((g,l)=>g-l)){let g=f&1048575,l=f>>>20;if(i[l]!==o)continue;let m=u.get(l);typeof m=="undefined"&&u.set(l,m=[]),m.push(g)}if(u.size===0)return s;let c=[];for(let[f,g]of u){let l=e[f],m=l[0]>>>12,x=l[l.length-1]>>>12,v=l[l.length-1]>>>2&1023;n&&m>a&&c.push(s.slice(a,m));let d=s.slice(m,x+v);for(let y of g.sort((b,E)=>E-b)){let b=(l[y]>>>12)-m,E=(l[y]>>>2&1023)+b;d=[d.slice(0,b),"<mark>",d.slice(b,E),"</mark>",d.slice(E)].join("")}if(a=x+v,c.push(d)===2)break}return n&&a<s.length&&c.push(s.slice(a)),c.join("")})}function oe(t){let e=[];if(typeof t=="undefined")return e;let r=Array.isArray(t)?t:[t];for(let n=0;n<r.length;n++){let i=lunr.tokenizer.table,s=i.length;ie(r[n],(o,a,u,c)=>{var f;switch(i[f=o+=s]||(i[f]=[]),a){case 0:case 2:i[o].push(u<<12|c-u<<2|a);break;case 1:let g=r[n].slice(u,c);W(g,lunr.tokenizer.separator,(l,m)=>{if(typeof lunr.segmenter!="undefined"){let x=g.slice(l,m);if(/^[MHIK]$/.test(lunr.segmenter.ctype_(x))){let v=lunr.segmenter.segment(x);for(let d=0,y=0;d<v.length;d++)i[o]||(i[o]=[]),i[o].push(u+l+y<<12|v[d].length<<2|a),e.push(new lunr.Token(v[d].toLowerCase(),{position:o<<20|i[o].length-1})),y+=v[d].length;return}}i[o].push(u+l<<12|m-l<<2|a),e.push(new lunr.Token(g.slice(l,m).toLowerCase(),{position:o<<20|i[o].length-1}))})}})}return e}function ae(t,e=r=>r){return t.trim().split(/"([^"]+)"/g).map((r,n)=>n&1?r.replace(/^\b|^(?![^\x00-\x7F]|$)|\s+/g," +"):r).join("").replace(/"|(?:^|\s+)[*+\-:^~]+(?=\s+|$)/g,"").split(/\s+/g).reduce((r,n)=>{let i=e(n);return[...r,...Array.isArray(i)?i:[i]]},[]).map(r=>/([~^]$)/.test(r)?`${r}1`:r).map(r=>/(^[+-]|[~^]\d+$)/.test(r)?r:`${r}*`).join(" ")}function ue(t){return ae(t,e=>{let r=[],n=new lunr.QueryLexer(e);n.run();for(let{type:i,str:s,start:o,end:a}of n.lexemes)switch(i){case"FIELD":["title","text","tags"].includes(s)||(e=[e.slice(0,a)," ",e.slice(a+1)].join(""));break;case"TERM":W(s,lunr.tokenizer.separator,(...u)=>{r.push([e.slice(0,o),s.slice(...u),e.slice(a)].join(""))})}return r})}function ce(t){let e=new lunr.Query(["title","text","tags"]);new lunr.QueryParser(t,e).parse();for(let n of e.clauses)n.usePipeline=!0,n.term.startsWith("*")&&(n.wildcard=lunr.Query.wildcard.LEADING,n.term=n.term.slice(1)),n.term.endsWith("*")&&(n.wildcard=lunr.Query.wildcard.TRAILING,n.term=n.term.slice(0,-1));return e.clauses}function le(t,e){var i;let r=new Set(t),n={};for(let s=0;s<e.length;s++)for(let o of r)e[s].startsWith(o.term)&&(n[o.term]=!0,r.delete(o));for(let s of r)(i=lunr.stopWordFilter)!=null&&i.call(lunr,s.term)&&(n[s.term]=!1);return n}function he(t,e){let r=new Set,n=new Uint16Array(t.length);for(let s=0;s<t.length;s++)for(let o=s+1;o<t.length;o++)t.slice(s,o)in e&&(n[s]=o-s);let i=[0];for(let s=i.length;s>0;){let o=i[--s];for(let u=1;u<n[o];u++)n[o+u]>n[o]-u&&(r.add(t.slice(o,o+u)),i[s++]=o+u);let a=o+n[o];n[a]&&a<t.length-1&&(i[s++]=a),r.add(t.slice(o,a))}return r.has("")?new Set([t]):r}function Oe(t){return e=>r=>{if(typeof r[e]=="undefined")return;let n=[r.location,e].join(":");return t.set(n,lunr.tokenizer.table=[]),r[e]}}function Re(t,e){let[r,n]=[new Set(t),new Set(e)];return[...new Set([...r].filter(i=>!n.has(i)))]}var H=class{constructor({config:e,docs:r,options:n}){let i=Oe(this.table=new Map);this.map=ne(r),this.options=n,this.index=lunr(function(){this.metadataWhitelist=["position"],this.b(0),e.lang.length===1&&e.lang[0]!=="en"?this.use(lunr[e.lang[0]]):e.lang.length>1&&this.use(lunr.multiLanguage(...e.lang)),this.tokenizer=oe,lunr.tokenizer.separator=new RegExp(e.separator),lunr.segmenter="TinySegmenter"in lunr?new lunr.TinySegmenter:void 0;let s=Re(["trimmer","stopWordFilter","stemmer"],e.pipeline);for(let o of e.lang.map(a=>a==="en"?lunr:lunr[a]))for(let a of s)this.pipeline.remove(o[a]),this.searchPipeline.remove(o[a]);this.ref("location"),this.field("title",{boost:1e3,extractor:i("title")}),this.field("text",{boost:1,extractor:i("text")}),this.field("tags",{boost:1e6,extractor:i("tags")});for(let o of r)this.add(o,{boost:o.boost})})}search(e){if(e=e.replace(new RegExp("\\p{sc=Han}+","gu"),s=>[...he(s,this.index.invertedIndex)].join("* ")),e=ue(e),!e)return{items:[]};let r=ce(e).filter(s=>s.presence!==lunr.Query.presence.PROHIBITED),n=this.index.search(e).reduce((s,{ref:o,score:a,matchData:u})=>{let c=this.map.get(o);if(typeof c!="undefined"){c=A({},c),c.tags&&(c.tags=[...c.tags]);let f=le(r,Object.keys(u.metadata));for(let l of this.index.fields){if(typeof c[l]=="undefined")continue;let m=[];for(let d of Object.values(u.metadata))typeof d[l]!="undefined"&&m.push(...d[l].position);if(!m.length)continue;let x=this.table.get([c.location,l].join(":")),v=Array.isArray(c[l])?q:se;c[l]=v(c[l],x,m,l!=="text")}let g=+!c.parent+Object.values(f).filter(l=>l).length/Object.keys(f).length;s.push(G(A({},c),{score:a*(1+Z(g,2)),terms:f}))}return s},[]).sort((s,o)=>o.score-s.score).reduce((s,o)=>{let a=this.map.get(o.location);if(typeof a!="undefined"){let u=a.parent?a.parent.location:a.location;s.set(u,[...s.get(u)||[],o])}return s},new Map);for(let[s,o]of n)if(!o.find(a=>a.location===s)){let a=this.map.get(s);o.push(G(A({},a),{score:0,terms:{}}))}let i;if(this.options.suggest){let s=this.index.query(o=>{for(let a of r)o.term(a.term,{fields:["title"],presence:lunr.Query.presence.REQUIRED,wildcard:lunr.Query.wildcard.TRAILING})});i=s.length?Object.keys(s[0].matchData.metadata):[]}return A({items:[...n.values()]},typeof i!="undefined"&&{suggest:i})}};var fe;function Ie(t){return B(this,null,function*(){let e="../lunr";if(typeof parent!="undefined"&&"IFrameWorker"in parent){let n=re("script[src]"),[i]=n.src.split("/worker");e=e.replace("..",i)}let r=[];for(let n of t.lang){switch(n){case"ja":r.push(`${e}/tinyseg.js`);break;case"hi":case"th":r.push(`${e}/wordcut.js`);break}n!=="en"&&r.push(`${e}/min/lunr.${n}.min.js`)}t.lang.length>1&&r.push(`${e}/min/lunr.multi.min.js`),r.length&&(yield importScripts(`${e}/min/lunr.stemmer.support.min.js`,...r))})}function Fe(t){return B(this,null,function*(){switch(t.type){case 0:return yield Ie(t.data.config),fe=new H(t.data),{type:1};case 2:let e=t.data;try{return{type:3,data:fe.search(e)}}catch(r){return console.warn(`Invalid query: ${e} \u2013 see https://bit.ly/2s3ChXG`),console.warn(r),{type:3,data:{items:[]}}}default:throw new TypeError("Invalid message type")}})}self.lunr=de.default;addEventListener("message",t=>B(void 0,null,function*(){postMessage(yield Fe(t.data))}));})();
+//# sourceMappingURL=search.b8dbb3d2.min.js.map
+
diff --git a/assets/javascripts/workers/search.b8dbb3d2.min.js.map b/assets/javascripts/workers/search.b8dbb3d2.min.js.map
new file mode 100644
index 000000000..7ef0846c0
--- /dev/null
+++ b/assets/javascripts/workers/search.b8dbb3d2.min.js.map
@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["node_modules/lunr/lunr.js", "src/templates/assets/javascripts/integrations/search/worker/main/index.ts", "src/templates/assets/javascripts/browser/element/_/index.ts", "src/templates/assets/javascripts/polyfills/index.ts", "src/templates/assets/javascripts/integrations/search/config/index.ts", "src/templates/assets/javascripts/integrations/search/internal/_/index.ts", "src/templates/assets/javascripts/integrations/search/internal/extract/index.ts", "src/templates/assets/javascripts/integrations/search/internal/highlight/index.ts", "src/templates/assets/javascripts/integrations/search/internal/tokenize/index.ts", "src/templates/assets/javascripts/integrations/search/query/transform/index.ts", "src/templates/assets/javascripts/integrations/search/query/_/index.ts", "src/templates/assets/javascripts/integrations/search/query/segment/index.ts", "src/templates/assets/javascripts/integrations/search/_/index.ts"],
+  "sourcesContent": ["/**\n * lunr - http://lunrjs.com - A bit like Solr, but much smaller and not as bright - 2.3.9\n * Copyright (C) 2020 Oliver Nightingale\n * @license MIT\n */\n\n;(function(){\n\n/**\n * A convenience function for configuring and constructing\n * a new lunr Index.\n *\n * A lunr.Builder instance is created and the pipeline setup\n * with a trimmer, stop word filter and stemmer.\n *\n * This builder object is yielded to the configuration function\n * that is passed as a parameter, allowing the list of fields\n * and other builder parameters to be customised.\n *\n * All documents _must_ be added within the passed config function.\n *\n * @example\n * var idx = lunr(function () {\n *   this.field('title')\n *   this.field('body')\n *   this.ref('id')\n *\n *   documents.forEach(function (doc) {\n *     this.add(doc)\n *   }, this)\n * })\n *\n * @see {@link lunr.Builder}\n * @see {@link lunr.Pipeline}\n * @see {@link lunr.trimmer}\n * @see {@link lunr.stopWordFilter}\n * @see {@link lunr.stemmer}\n * @namespace {function} lunr\n */\nvar lunr = function (config) {\n  var builder = new lunr.Builder\n\n  builder.pipeline.add(\n    lunr.trimmer,\n    lunr.stopWordFilter,\n    lunr.stemmer\n  )\n\n  builder.searchPipeline.add(\n    lunr.stemmer\n  )\n\n  config.call(builder, builder)\n  return builder.build()\n}\n\nlunr.version = \"2.3.9\"\n/*!\n * lunr.utils\n * Copyright (C) 2020 Oliver Nightingale\n */\n\n/**\n * A namespace containing utils for the rest of the lunr library\n * @namespace lunr.utils\n */\nlunr.utils = {}\n\n/**\n * Print a warning message to the console.\n *\n * @param {String} message The message to be printed.\n * @memberOf lunr.utils\n * @function\n */\nlunr.utils.warn = (function (global) {\n  /* eslint-disable no-console */\n  return function (message) {\n    if (global.console && console.warn) {\n      console.warn(message)\n    }\n  }\n  /* eslint-enable no-console */\n})(this)\n\n/**\n * Convert an object to a string.\n *\n * In the case of `null` and `undefined` the function returns\n * the empty string, in all other cases the result of calling\n * `toString` on the passed object is returned.\n *\n * @param {Any} obj The object to convert to a string.\n * @return {String} string representation of the passed object.\n * @memberOf lunr.utils\n */\nlunr.utils.asString = function (obj) {\n  if (obj === void 0 || obj === null) {\n    return \"\"\n  } else {\n    return obj.toString()\n  }\n}\n\n/**\n * Clones an object.\n *\n * Will create a copy of an existing object such that any mutations\n * on the copy cannot affect the original.\n *\n * Only shallow objects are supported, passing a nested object to this\n * function will cause a TypeError.\n *\n * Objects with primitives, and arrays of primitives are supported.\n *\n * @param {Object} obj The object to clone.\n * @return {Object} a clone of the passed object.\n * @throws {TypeError} when a nested object is passed.\n * @memberOf Utils\n */\nlunr.utils.clone = function (obj) {\n  if (obj === null || obj === undefined) {\n    return obj\n  }\n\n  var clone = Object.create(null),\n      keys = Object.keys(obj)\n\n  for (var i = 0; i < keys.length; i++) {\n    var key = keys[i],\n        val = obj[key]\n\n    if (Array.isArray(val)) {\n      clone[key] = val.slice()\n      continue\n    }\n\n    if (typeof val === 'string' ||\n        typeof val === 'number' ||\n        typeof val === 'boolean') {\n      clone[key] = val\n      continue\n    }\n\n    throw new TypeError(\"clone is not deep and does not support nested objects\")\n  }\n\n  return clone\n}\nlunr.FieldRef = function (docRef, fieldName, stringValue) {\n  this.docRef = docRef\n  this.fieldName = fieldName\n  this._stringValue = stringValue\n}\n\nlunr.FieldRef.joiner = \"/\"\n\nlunr.FieldRef.fromString = function (s) {\n  var n = s.indexOf(lunr.FieldRef.joiner)\n\n  if (n === -1) {\n    throw \"malformed field ref string\"\n  }\n\n  var fieldRef = s.slice(0, n),\n      docRef = s.slice(n + 1)\n\n  return new lunr.FieldRef (docRef, fieldRef, s)\n}\n\nlunr.FieldRef.prototype.toString = function () {\n  if (this._stringValue == undefined) {\n    this._stringValue = this.fieldName + lunr.FieldRef.joiner + this.docRef\n  }\n\n  return this._stringValue\n}\n/*!\n * lunr.Set\n * Copyright (C) 2020 Oliver Nightingale\n */\n\n/**\n * A lunr set.\n *\n * @constructor\n */\nlunr.Set = function (elements) {\n  this.elements = Object.create(null)\n\n  if (elements) {\n    this.length = elements.length\n\n    for (var i = 0; i < this.length; i++) {\n      this.elements[elements[i]] = true\n    }\n  } else {\n    this.length = 0\n  }\n}\n\n/**\n * A complete set that contains all elements.\n *\n * @static\n * @readonly\n * @type {lunr.Set}\n */\nlunr.Set.complete = {\n  intersect: function (other) {\n    return other\n  },\n\n  union: function () {\n    return this\n  },\n\n  contains: function () {\n    return true\n  }\n}\n\n/**\n * An empty set that contains no elements.\n *\n * @static\n * @readonly\n * @type {lunr.Set}\n */\nlunr.Set.empty = {\n  intersect: function () {\n    return this\n  },\n\n  union: function (other) {\n    return other\n  },\n\n  contains: function () {\n    return false\n  }\n}\n\n/**\n * Returns true if this set contains the specified object.\n *\n * @param {object} object - Object whose presence in this set is to be tested.\n * @returns {boolean} - True if this set contains the specified object.\n */\nlunr.Set.prototype.contains = function (object) {\n  return !!this.elements[object]\n}\n\n/**\n * Returns a new set containing only the elements that are present in both\n * this set and the specified set.\n *\n * @param {lunr.Set} other - set to intersect with this set.\n * @returns {lunr.Set} a new set that is the intersection of this and the specified set.\n */\n\nlunr.Set.prototype.intersect = function (other) {\n  var a, b, elements, intersection = []\n\n  if (other === lunr.Set.complete) {\n    return this\n  }\n\n  if (other === lunr.Set.empty) {\n    return other\n  }\n\n  if (this.length < other.length) {\n    a = this\n    b = other\n  } else {\n    a = other\n    b = this\n  }\n\n  elements = Object.keys(a.elements)\n\n  for (var i = 0; i < elements.length; i++) {\n    var element = elements[i]\n    if (element in b.elements) {\n      intersection.push(element)\n    }\n  }\n\n  return new lunr.Set (intersection)\n}\n\n/**\n * Returns a new set combining the elements of this and the specified set.\n *\n * @param {lunr.Set} other - set to union with this set.\n * @return {lunr.Set} a new set that is the union of this and the specified set.\n */\n\nlunr.Set.prototype.union = function (other) {\n  if (other === lunr.Set.complete) {\n    return lunr.Set.complete\n  }\n\n  if (other === lunr.Set.empty) {\n    return this\n  }\n\n  return new lunr.Set(Object.keys(this.elements).concat(Object.keys(other.elements)))\n}\n/**\n * A function to calculate the inverse document frequency for\n * a posting. This is shared between the builder and the index\n *\n * @private\n * @param {object} posting - The posting for a given term\n * @param {number} documentCount - The total number of documents.\n */\nlunr.idf = function (posting, documentCount) {\n  var documentsWithTerm = 0\n\n  for (var fieldName in posting) {\n    if (fieldName == '_index') continue // Ignore the term index, its not a field\n    documentsWithTerm += Object.keys(posting[fieldName]).length\n  }\n\n  var x = (documentCount - documentsWithTerm + 0.5) / (documentsWithTerm + 0.5)\n\n  return Math.log(1 + Math.abs(x))\n}\n\n/**\n * A token wraps a string representation of a token\n * as it is passed through the text processing pipeline.\n *\n * @constructor\n * @param {string} [str=''] - The string token being wrapped.\n * @param {object} [metadata={}] - Metadata associated with this token.\n */\nlunr.Token = function (str, metadata) {\n  this.str = str || \"\"\n  this.metadata = metadata || {}\n}\n\n/**\n * Returns the token string that is being wrapped by this object.\n *\n * @returns {string}\n */\nlunr.Token.prototype.toString = function () {\n  return this.str\n}\n\n/**\n * A token update function is used when updating or optionally\n * when cloning a token.\n *\n * @callback lunr.Token~updateFunction\n * @param {string} str - The string representation of the token.\n * @param {Object} metadata - All metadata associated with this token.\n */\n\n/**\n * Applies the given function to the wrapped string token.\n *\n * @example\n * token.update(function (str, metadata) {\n *   return str.toUpperCase()\n * })\n *\n * @param {lunr.Token~updateFunction} fn - A function to apply to the token string.\n * @returns {lunr.Token}\n */\nlunr.Token.prototype.update = function (fn) {\n  this.str = fn(this.str, this.metadata)\n  return this\n}\n\n/**\n * Creates a clone of this token. Optionally a function can be\n * applied to the cloned token.\n *\n * @param {lunr.Token~updateFunction} [fn] - An optional function to apply to the cloned token.\n * @returns {lunr.Token}\n */\nlunr.Token.prototype.clone = function (fn) {\n  fn = fn || function (s) { return s }\n  return new lunr.Token (fn(this.str, this.metadata), this.metadata)\n}\n/*!\n * lunr.tokenizer\n * Copyright (C) 2020 Oliver Nightingale\n */\n\n/**\n * A function for splitting a string into tokens ready to be inserted into\n * the search index. Uses `lunr.tokenizer.separator` to split strings, change\n * the value of this property to change how strings are split into tokens.\n *\n * This tokenizer will convert its parameter to a string by calling `toString` and\n * then will split this string on the character in `lunr.tokenizer.separator`.\n * Arrays will have their elements converted to strings and wrapped in a lunr.Token.\n *\n * Optional metadata can be passed to the tokenizer, this metadata will be cloned and\n * added as metadata to every token that is created from the object to be tokenized.\n *\n * @static\n * @param {?(string|object|object[])} obj - The object to convert into tokens\n * @param {?object} metadata - Optional metadata to associate with every token\n * @returns {lunr.Token[]}\n * @see {@link lunr.Pipeline}\n */\nlunr.tokenizer = function (obj, metadata) {\n  if (obj == null || obj == undefined) {\n    return []\n  }\n\n  if (Array.isArray(obj)) {\n    return obj.map(function (t) {\n      return new lunr.Token(\n        lunr.utils.asString(t).toLowerCase(),\n        lunr.utils.clone(metadata)\n      )\n    })\n  }\n\n  var str = obj.toString().toLowerCase(),\n      len = str.length,\n      tokens = []\n\n  for (var sliceEnd = 0, sliceStart = 0; sliceEnd <= len; sliceEnd++) {\n    var char = str.charAt(sliceEnd),\n        sliceLength = sliceEnd - sliceStart\n\n    if ((char.match(lunr.tokenizer.separator) || sliceEnd == len)) {\n\n      if (sliceLength > 0) {\n        var tokenMetadata = lunr.utils.clone(metadata) || {}\n        tokenMetadata[\"position\"] = [sliceStart, sliceLength]\n        tokenMetadata[\"index\"] = tokens.length\n\n        tokens.push(\n          new lunr.Token (\n            str.slice(sliceStart, sliceEnd),\n            tokenMetadata\n          )\n        )\n      }\n\n      sliceStart = sliceEnd + 1\n    }\n\n  }\n\n  return tokens\n}\n\n/**\n * The separator used to split a string into tokens. Override this property to change the behaviour of\n * `lunr.tokenizer` behaviour when tokenizing strings. By default this splits on whitespace and hyphens.\n *\n * @static\n * @see lunr.tokenizer\n */\nlunr.tokenizer.separator = /[\\s\\-]+/\n/*!\n * lunr.Pipeline\n * Copyright (C) 2020 Oliver Nightingale\n */\n\n/**\n * lunr.Pipelines maintain an ordered list of functions to be applied to all\n * tokens in documents entering the search index and queries being ran against\n * the index.\n *\n * An instance of lunr.Index created with the lunr shortcut will contain a\n * pipeline with a stop word filter and an English language stemmer. Extra\n * functions can be added before or after either of these functions or these\n * default functions can be removed.\n *\n * When run the pipeline will call each function in turn, passing a token, the\n * index of that token in the original list of all tokens and finally a list of\n * all the original tokens.\n *\n * The output of functions in the pipeline will be passed to the next function\n * in the pipeline. To exclude a token from entering the index the function\n * should return undefined, the rest of the pipeline will not be called with\n * this token.\n *\n * For serialisation of pipelines to work, all functions used in an instance of\n * a pipeline should be registered with lunr.Pipeline. Registered functions can\n * then be loaded. If trying to load a serialised pipeline that uses functions\n * that are not registered an error will be thrown.\n *\n * If not planning on serialising the pipeline then registering pipeline functions\n * is not necessary.\n *\n * @constructor\n */\nlunr.Pipeline = function () {\n  this._stack = []\n}\n\nlunr.Pipeline.registeredFunctions = Object.create(null)\n\n/**\n * A pipeline function maps lunr.Token to lunr.Token. A lunr.Token contains the token\n * string as well as all known metadata. A pipeline function can mutate the token string\n * or mutate (or add) metadata for a given token.\n *\n * A pipeline function can indicate that the passed token should be discarded by returning\n * null, undefined or an empty string. This token will not be passed to any downstream pipeline\n * functions and will not be added to the index.\n *\n * Multiple tokens can be returned by returning an array of tokens. Each token will be passed\n * to any downstream pipeline functions and all will returned tokens will be added to the index.\n *\n * Any number of pipeline functions may be chained together using a lunr.Pipeline.\n *\n * @interface lunr.PipelineFunction\n * @param {lunr.Token} token - A token from the document being processed.\n * @param {number} i - The index of this token in the complete list of tokens for this document/field.\n * @param {lunr.Token[]} tokens - All tokens for this document/field.\n * @returns {(?lunr.Token|lunr.Token[])}\n */\n\n/**\n * Register a function with the pipeline.\n *\n * Functions that are used in the pipeline should be registered if the pipeline\n * needs to be serialised, or a serialised pipeline needs to be loaded.\n *\n * Registering a function does not add it to a pipeline, functions must still be\n * added to instances of the pipeline for them to be used when running a pipeline.\n *\n * @param {lunr.PipelineFunction} fn - The function to check for.\n * @param {String} label - The label to register this function with\n */\nlunr.Pipeline.registerFunction = function (fn, label) {\n  if (label in this.registeredFunctions) {\n    lunr.utils.warn('Overwriting existing registered function: ' + label)\n  }\n\n  fn.label = label\n  lunr.Pipeline.registeredFunctions[fn.label] = fn\n}\n\n/**\n * Warns if the function is not registered as a Pipeline function.\n *\n * @param {lunr.PipelineFunction} fn - The function to check for.\n * @private\n */\nlunr.Pipeline.warnIfFunctionNotRegistered = function (fn) {\n  var isRegistered = fn.label && (fn.label in this.registeredFunctions)\n\n  if (!isRegistered) {\n    lunr.utils.warn('Function is not registered with pipeline. This may cause problems when serialising the index.\\n', fn)\n  }\n}\n\n/**\n * Loads a previously serialised pipeline.\n *\n * All functions to be loaded must already be registered with lunr.Pipeline.\n * If any function from the serialised data has not been registered then an\n * error will be thrown.\n *\n * @param {Object} serialised - The serialised pipeline to load.\n * @returns {lunr.Pipeline}\n */\nlunr.Pipeline.load = function (serialised) {\n  var pipeline = new lunr.Pipeline\n\n  serialised.forEach(function (fnName) {\n    var fn = lunr.Pipeline.registeredFunctions[fnName]\n\n    if (fn) {\n      pipeline.add(fn)\n    } else {\n      throw new Error('Cannot load unregistered function: ' + fnName)\n    }\n  })\n\n  return pipeline\n}\n\n/**\n * Adds new functions to the end of the pipeline.\n *\n * Logs a warning if the function has not been registered.\n *\n * @param {lunr.PipelineFunction[]} functions - Any number of functions to add to the pipeline.\n */\nlunr.Pipeline.prototype.add = function () {\n  var fns = Array.prototype.slice.call(arguments)\n\n  fns.forEach(function (fn) {\n    lunr.Pipeline.warnIfFunctionNotRegistered(fn)\n    this._stack.push(fn)\n  }, this)\n}\n\n/**\n * Adds a single function after a function that already exists in the\n * pipeline.\n *\n * Logs a warning if the function has not been registered.\n *\n * @param {lunr.PipelineFunction} existingFn - A function that already exists in the pipeline.\n * @param {lunr.PipelineFunction} newFn - The new function to add to the pipeline.\n */\nlunr.Pipeline.prototype.after = function (existingFn, newFn) {\n  lunr.Pipeline.warnIfFunctionNotRegistered(newFn)\n\n  var pos = this._stack.indexOf(existingFn)\n  if (pos == -1) {\n    throw new Error('Cannot find existingFn')\n  }\n\n  pos = pos + 1\n  this._stack.splice(pos, 0, newFn)\n}\n\n/**\n * Adds a single function before a function that already exists in the\n * pipeline.\n *\n * Logs a warning if the function has not been registered.\n *\n * @param {lunr.PipelineFunction} existingFn - A function that already exists in the pipeline.\n * @param {lunr.PipelineFunction} newFn - The new function to add to the pipeline.\n */\nlunr.Pipeline.prototype.before = function (existingFn, newFn) {\n  lunr.Pipeline.warnIfFunctionNotRegistered(newFn)\n\n  var pos = this._stack.indexOf(existingFn)\n  if (pos == -1) {\n    throw new Error('Cannot find existingFn')\n  }\n\n  this._stack.splice(pos, 0, newFn)\n}\n\n/**\n * Removes a function from the pipeline.\n *\n * @param {lunr.PipelineFunction} fn The function to remove from the pipeline.\n */\nlunr.Pipeline.prototype.remove = function (fn) {\n  var pos = this._stack.indexOf(fn)\n  if (pos == -1) {\n    return\n  }\n\n  this._stack.splice(pos, 1)\n}\n\n/**\n * Runs the current list of functions that make up the pipeline against the\n * passed tokens.\n *\n * @param {Array} tokens The tokens to run through the pipeline.\n * @returns {Array}\n */\nlunr.Pipeline.prototype.run = function (tokens) {\n  var stackLength = this._stack.length\n\n  for (var i = 0; i < stackLength; i++) {\n    var fn = this._stack[i]\n    var memo = []\n\n    for (var j = 0; j < tokens.length; j++) {\n      var result = fn(tokens[j], j, tokens)\n\n      if (result === null || result === void 0 || result === '') continue\n\n      if (Array.isArray(result)) {\n        for (var k = 0; k < result.length; k++) {\n          memo.push(result[k])\n        }\n      } else {\n        memo.push(result)\n      }\n    }\n\n    tokens = memo\n  }\n\n  return tokens\n}\n\n/**\n * Convenience method for passing a string through a pipeline and getting\n * strings out. This method takes care of wrapping the passed string in a\n * token and mapping the resulting tokens back to strings.\n *\n * @param {string} str - The string to pass through the pipeline.\n * @param {?object} metadata - Optional metadata to associate with the token\n * passed to the pipeline.\n * @returns {string[]}\n */\nlunr.Pipeline.prototype.runString = function (str, metadata) {\n  var token = new lunr.Token (str, metadata)\n\n  return this.run([token]).map(function (t) {\n    return t.toString()\n  })\n}\n\n/**\n * Resets the pipeline by removing any existing processors.\n *\n */\nlunr.Pipeline.prototype.reset = function () {\n  this._stack = []\n}\n\n/**\n * Returns a representation of the pipeline ready for serialisation.\n *\n * Logs a warning if the function has not been registered.\n *\n * @returns {Array}\n */\nlunr.Pipeline.prototype.toJSON = function () {\n  return this._stack.map(function (fn) {\n    lunr.Pipeline.warnIfFunctionNotRegistered(fn)\n\n    return fn.label\n  })\n}\n/*!\n * lunr.Vector\n * Copyright (C) 2020 Oliver Nightingale\n */\n\n/**\n * A vector is used to construct the vector space of documents and queries. These\n * vectors support operations to determine the similarity between two documents or\n * a document and a query.\n *\n * Normally no parameters are required for initializing a vector, but in the case of\n * loading a previously dumped vector the raw elements can be provided to the constructor.\n *\n * For performance reasons vectors are implemented with a flat array, where an elements\n * index is immediately followed by its value. E.g. [index, value, index, value]. This\n * allows the underlying array to be as sparse as possible and still offer decent\n * performance when being used for vector calculations.\n *\n * @constructor\n * @param {Number[]} [elements] - The flat list of element index and element value pairs.\n */\nlunr.Vector = function (elements) {\n  this._magnitude = 0\n  this.elements = elements || []\n}\n\n\n/**\n * Calculates the position within the vector to insert a given index.\n *\n * This is used internally by insert and upsert. If there are duplicate indexes then\n * the position is returned as if the value for that index were to be updated, but it\n * is the callers responsibility to check whether there is a duplicate at that index\n *\n * @param {Number} insertIdx - The index at which the element should be inserted.\n * @returns {Number}\n */\nlunr.Vector.prototype.positionForIndex = function (index) {\n  // For an empty vector the tuple can be inserted at the beginning\n  if (this.elements.length == 0) {\n    return 0\n  }\n\n  var start = 0,\n      end = this.elements.length / 2,\n      sliceLength = end - start,\n      pivotPoint = Math.floor(sliceLength / 2),\n      pivotIndex = this.elements[pivotPoint * 2]\n\n  while (sliceLength > 1) {\n    if (pivotIndex < index) {\n      start = pivotPoint\n    }\n\n    if (pivotIndex > index) {\n      end = pivotPoint\n    }\n\n    if (pivotIndex == index) {\n      break\n    }\n\n    sliceLength = end - start\n    pivotPoint = start + Math.floor(sliceLength / 2)\n    pivotIndex = this.elements[pivotPoint * 2]\n  }\n\n  if (pivotIndex == index) {\n    return pivotPoint * 2\n  }\n\n  if (pivotIndex > index) {\n    return pivotPoint * 2\n  }\n\n  if (pivotIndex < index) {\n    return (pivotPoint + 1) * 2\n  }\n}\n\n/**\n * Inserts an element at an index within the vector.\n *\n * Does not allow duplicates, will throw an error if there is already an entry\n * for this index.\n *\n * @param {Number} insertIdx - The index at which the element should be inserted.\n * @param {Number} val - The value to be inserted into the vector.\n */\nlunr.Vector.prototype.insert = function (insertIdx, val) {\n  this.upsert(insertIdx, val, function () {\n    throw \"duplicate index\"\n  })\n}\n\n/**\n * Inserts or updates an existing index within the vector.\n *\n * @param {Number} insertIdx - The index at which the element should be inserted.\n * @param {Number} val - The value to be inserted into the vector.\n * @param {function} fn - A function that is called for updates, the existing value and the\n * requested value are passed as arguments\n */\nlunr.Vector.prototype.upsert = function (insertIdx, val, fn) {\n  this._magnitude = 0\n  var position = this.positionForIndex(insertIdx)\n\n  if (this.elements[position] == insertIdx) {\n    this.elements[position + 1] = fn(this.elements[position + 1], val)\n  } else {\n    this.elements.splice(position, 0, insertIdx, val)\n  }\n}\n\n/**\n * Calculates the magnitude of this vector.\n *\n * @returns {Number}\n */\nlunr.Vector.prototype.magnitude = function () {\n  if (this._magnitude) return this._magnitude\n\n  var sumOfSquares = 0,\n      elementsLength = this.elements.length\n\n  for (var i = 1; i < elementsLength; i += 2) {\n    var val = this.elements[i]\n    sumOfSquares += val * val\n  }\n\n  return this._magnitude = Math.sqrt(sumOfSquares)\n}\n\n/**\n * Calculates the dot product of this vector and another vector.\n *\n * @param {lunr.Vector} otherVector - The vector to compute the dot product with.\n * @returns {Number}\n */\nlunr.Vector.prototype.dot = function (otherVector) {\n  var dotProduct = 0,\n      a = this.elements, b = otherVector.elements,\n      aLen = a.length, bLen = b.length,\n      aVal = 0, bVal = 0,\n      i = 0, j = 0\n\n  while (i < aLen && j < bLen) {\n    aVal = a[i], bVal = b[j]\n    if (aVal < bVal) {\n      i += 2\n    } else if (aVal > bVal) {\n      j += 2\n    } else if (aVal == bVal) {\n      dotProduct += a[i + 1] * b[j + 1]\n      i += 2\n      j += 2\n    }\n  }\n\n  return dotProduct\n}\n\n/**\n * Calculates the similarity between this vector and another vector.\n *\n * @param {lunr.Vector} otherVector - The other vector to calculate the\n * similarity with.\n * @returns {Number}\n */\nlunr.Vector.prototype.similarity = function (otherVector) {\n  return this.dot(otherVector) / this.magnitude() || 0\n}\n\n/**\n * Converts the vector to an array of the elements within the vector.\n *\n * @returns {Number[]}\n */\nlunr.Vector.prototype.toArray = function () {\n  var output = new Array (this.elements.length / 2)\n\n  for (var i = 1, j = 0; i < this.elements.length; i += 2, j++) {\n    output[j] = this.elements[i]\n  }\n\n  return output\n}\n\n/**\n * A JSON serializable representation of the vector.\n *\n * @returns {Number[]}\n */\nlunr.Vector.prototype.toJSON = function () {\n  return this.elements\n}\n/* eslint-disable */\n/*!\n * lunr.stemmer\n * Copyright (C) 2020 Oliver Nightingale\n * Includes code from - http://tartarus.org/~martin/PorterStemmer/js.txt\n */\n\n/**\n * lunr.stemmer is an english language stemmer, this is a JavaScript\n * implementation of the PorterStemmer taken from http://tartarus.org/~martin\n *\n * @static\n * @implements {lunr.PipelineFunction}\n * @param {lunr.Token} token - The string to stem\n * @returns {lunr.Token}\n * @see {@link lunr.Pipeline}\n * @function\n */\nlunr.stemmer = (function(){\n  var step2list = {\n      \"ational\" : \"ate\",\n      \"tional\" : \"tion\",\n      \"enci\" : \"ence\",\n      \"anci\" : \"ance\",\n      \"izer\" : \"ize\",\n      \"bli\" : \"ble\",\n      \"alli\" : \"al\",\n      \"entli\" : \"ent\",\n      \"eli\" : \"e\",\n      \"ousli\" : \"ous\",\n      \"ization\" : \"ize\",\n      \"ation\" : \"ate\",\n      \"ator\" : \"ate\",\n      \"alism\" : \"al\",\n      \"iveness\" : \"ive\",\n      \"fulness\" : \"ful\",\n      \"ousness\" : \"ous\",\n      \"aliti\" : \"al\",\n      \"iviti\" : \"ive\",\n      \"biliti\" : \"ble\",\n      \"logi\" : \"log\"\n    },\n\n    step3list = {\n      \"icate\" : \"ic\",\n      \"ative\" : \"\",\n      \"alize\" : \"al\",\n      \"iciti\" : \"ic\",\n      \"ical\" : \"ic\",\n      \"ful\" : \"\",\n      \"ness\" : \"\"\n    },\n\n    c = \"[^aeiou]\",          // consonant\n    v = \"[aeiouy]\",          // vowel\n    C = c + \"[^aeiouy]*\",    // consonant sequence\n    V = v + \"[aeiou]*\",      // vowel sequence\n\n    mgr0 = \"^(\" + C + \")?\" + V + C,               // [C]VC... is m>0\n    meq1 = \"^(\" + C + \")?\" + V + C + \"(\" + V + \")?$\",  // [C]VC[V] is m=1\n    mgr1 = \"^(\" + C + \")?\" + V + C + V + C,       // [C]VCVC... is m>1\n    s_v = \"^(\" + C + \")?\" + v;                   // vowel in stem\n\n  var re_mgr0 = new RegExp(mgr0);\n  var re_mgr1 = new RegExp(mgr1);\n  var re_meq1 = new RegExp(meq1);\n  var re_s_v = new RegExp(s_v);\n\n  var re_1a = /^(.+?)(ss|i)es$/;\n  var re2_1a = /^(.+?)([^s])s$/;\n  var re_1b = /^(.+?)eed$/;\n  var re2_1b = /^(.+?)(ed|ing)$/;\n  var re_1b_2 = /.$/;\n  var re2_1b_2 = /(at|bl|iz)$/;\n  var re3_1b_2 = new RegExp(\"([^aeiouylsz])\\\\1$\");\n  var re4_1b_2 = new RegExp(\"^\" + C + v + \"[^aeiouwxy]$\");\n\n  var re_1c = /^(.+?[^aeiou])y$/;\n  var re_2 = /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/;\n\n  var re_3 = /^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/;\n\n  var re_4 = /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/;\n  var re2_4 = /^(.+?)(s|t)(ion)$/;\n\n  var re_5 = /^(.+?)e$/;\n  var re_5_1 = /ll$/;\n  var re3_5 = new RegExp(\"^\" + C + v + \"[^aeiouwxy]$\");\n\n  var porterStemmer = function porterStemmer(w) {\n    var stem,\n      suffix,\n      firstch,\n      re,\n      re2,\n      re3,\n      re4;\n\n    if (w.length < 3) { return w; }\n\n    firstch = w.substr(0,1);\n    if (firstch == \"y\") {\n      w = firstch.toUpperCase() + w.substr(1);\n    }\n\n    // Step 1a\n    re = re_1a\n    re2 = re2_1a;\n\n    if (re.test(w)) { w = w.replace(re,\"$1$2\"); }\n    else if (re2.test(w)) { w = w.replace(re2,\"$1$2\"); }\n\n    // Step 1b\n    re = re_1b;\n    re2 = re2_1b;\n    if (re.test(w)) {\n      var fp = re.exec(w);\n      re = re_mgr0;\n      if (re.test(fp[1])) {\n        re = re_1b_2;\n        w = w.replace(re,\"\");\n      }\n    } else if (re2.test(w)) {\n      var fp = re2.exec(w);\n      stem = fp[1];\n      re2 = re_s_v;\n      if (re2.test(stem)) {\n        w = stem;\n        re2 = re2_1b_2;\n        re3 = re3_1b_2;\n        re4 = re4_1b_2;\n        if (re2.test(w)) { w = w + \"e\"; }\n        else if (re3.test(w)) { re = re_1b_2; w = w.replace(re,\"\"); }\n        else if (re4.test(w)) { w = w + \"e\"; }\n      }\n    }\n\n    // Step 1c - replace suffix y or Y by i if preceded by a non-vowel which is not the first letter of the word (so cry -> cri, by -> by, say -> say)\n    re = re_1c;\n    if (re.test(w)) {\n      var fp = re.exec(w);\n      stem = fp[1];\n      w = stem + \"i\";\n    }\n\n    // Step 2\n    re = re_2;\n    if (re.test(w)) {\n      var fp = re.exec(w);\n      stem = fp[1];\n      suffix = fp[2];\n      re = re_mgr0;\n      if (re.test(stem)) {\n        w = stem + step2list[suffix];\n      }\n    }\n\n    // Step 3\n    re = re_3;\n    if (re.test(w)) {\n      var fp = re.exec(w);\n      stem = fp[1];\n      suffix = fp[2];\n      re = re_mgr0;\n      if (re.test(stem)) {\n        w = stem + step3list[suffix];\n      }\n    }\n\n    // Step 4\n    re = re_4;\n    re2 = re2_4;\n    if (re.test(w)) {\n      var fp = re.exec(w);\n      stem = fp[1];\n      re = re_mgr1;\n      if (re.test(stem)) {\n        w = stem;\n      }\n    } else if (re2.test(w)) {\n      var fp = re2.exec(w);\n      stem = fp[1] + fp[2];\n      re2 = re_mgr1;\n      if (re2.test(stem)) {\n        w = stem;\n      }\n    }\n\n    // Step 5\n    re = re_5;\n    if (re.test(w)) {\n      var fp = re.exec(w);\n      stem = fp[1];\n      re = re_mgr1;\n      re2 = re_meq1;\n      re3 = re3_5;\n      if (re.test(stem) || (re2.test(stem) && !(re3.test(stem)))) {\n        w = stem;\n      }\n    }\n\n    re = re_5_1;\n    re2 = re_mgr1;\n    if (re.test(w) && re2.test(w)) {\n      re = re_1b_2;\n      w = w.replace(re,\"\");\n    }\n\n    // and turn initial Y back to y\n\n    if (firstch == \"y\") {\n      w = firstch.toLowerCase() + w.substr(1);\n    }\n\n    return w;\n  };\n\n  return function (token) {\n    return token.update(porterStemmer);\n  }\n})();\n\nlunr.Pipeline.registerFunction(lunr.stemmer, 'stemmer')\n/*!\n * lunr.stopWordFilter\n * Copyright (C) 2020 Oliver Nightingale\n */\n\n/**\n * lunr.generateStopWordFilter builds a stopWordFilter function from the provided\n * list of stop words.\n *\n * The built in lunr.stopWordFilter is built using this generator and can be used\n * to generate custom stopWordFilters for applications or non English languages.\n *\n * @function\n * @param {Array} token The token to pass through the filter\n * @returns {lunr.PipelineFunction}\n * @see lunr.Pipeline\n * @see lunr.stopWordFilter\n */\nlunr.generateStopWordFilter = function (stopWords) {\n  var words = stopWords.reduce(function (memo, stopWord) {\n    memo[stopWord] = stopWord\n    return memo\n  }, {})\n\n  return function (token) {\n    if (token && words[token.toString()] !== token.toString()) return token\n  }\n}\n\n/**\n * lunr.stopWordFilter is an English language stop word list filter, any words\n * contained in the list will not be passed through the filter.\n *\n * This is intended to be used in the Pipeline. If the token does not pass the\n * filter then undefined will be returned.\n *\n * @function\n * @implements {lunr.PipelineFunction}\n * @params {lunr.Token} token - A token to check for being a stop word.\n * @returns {lunr.Token}\n * @see {@link lunr.Pipeline}\n */\nlunr.stopWordFilter = lunr.generateStopWordFilter([\n  'a',\n  'able',\n  'about',\n  'across',\n  'after',\n  'all',\n  'almost',\n  'also',\n  'am',\n  'among',\n  'an',\n  'and',\n  'any',\n  'are',\n  'as',\n  'at',\n  'be',\n  'because',\n  'been',\n  'but',\n  'by',\n  'can',\n  'cannot',\n  'could',\n  'dear',\n  'did',\n  'do',\n  'does',\n  'either',\n  'else',\n  'ever',\n  'every',\n  'for',\n  'from',\n  'get',\n  'got',\n  'had',\n  'has',\n  'have',\n  'he',\n  'her',\n  'hers',\n  'him',\n  'his',\n  'how',\n  'however',\n  'i',\n  'if',\n  'in',\n  'into',\n  'is',\n  'it',\n  'its',\n  'just',\n  'least',\n  'let',\n  'like',\n  'likely',\n  'may',\n  'me',\n  'might',\n  'most',\n  'must',\n  'my',\n  'neither',\n  'no',\n  'nor',\n  'not',\n  'of',\n  'off',\n  'often',\n  'on',\n  'only',\n  'or',\n  'other',\n  'our',\n  'own',\n  'rather',\n  'said',\n  'say',\n  'says',\n  'she',\n  'should',\n  'since',\n  'so',\n  'some',\n  'than',\n  'that',\n  'the',\n  'their',\n  'them',\n  'then',\n  'there',\n  'these',\n  'they',\n  'this',\n  'tis',\n  'to',\n  'too',\n  'twas',\n  'us',\n  'wants',\n  'was',\n  'we',\n  'were',\n  'what',\n  'when',\n  'where',\n  'which',\n  'while',\n  'who',\n  'whom',\n  'why',\n  'will',\n  'with',\n  'would',\n  'yet',\n  'you',\n  'your'\n])\n\nlunr.Pipeline.registerFunction(lunr.stopWordFilter, 'stopWordFilter')\n/*!\n * lunr.trimmer\n * Copyright (C) 2020 Oliver Nightingale\n */\n\n/**\n * lunr.trimmer is a pipeline function for trimming non word\n * characters from the beginning and end of tokens before they\n * enter the index.\n *\n * This implementation may not work correctly for non latin\n * characters and should either be removed or adapted for use\n * with languages with non-latin characters.\n *\n * @static\n * @implements {lunr.PipelineFunction}\n * @param {lunr.Token} token The token to pass through the filter\n * @returns {lunr.Token}\n * @see lunr.Pipeline\n */\nlunr.trimmer = function (token) {\n  return token.update(function (s) {\n    return s.replace(/^\\W+/, '').replace(/\\W+$/, '')\n  })\n}\n\nlunr.Pipeline.registerFunction(lunr.trimmer, 'trimmer')\n/*!\n * lunr.TokenSet\n * Copyright (C) 2020 Oliver Nightingale\n */\n\n/**\n * A token set is used to store the unique list of all tokens\n * within an index. Token sets are also used to represent an\n * incoming query to the index, this query token set and index\n * token set are then intersected to find which tokens to look\n * up in the inverted index.\n *\n * A token set can hold multiple tokens, as in the case of the\n * index token set, or it can hold a single token as in the\n * case of a simple query token set.\n *\n * Additionally token sets are used to perform wildcard matching.\n * Leading, contained and trailing wildcards are supported, and\n * from this edit distance matching can also be provided.\n *\n * Token sets are implemented as a minimal finite state automata,\n * where both common prefixes and suffixes are shared between tokens.\n * This helps to reduce the space used for storing the token set.\n *\n * @constructor\n */\nlunr.TokenSet = function () {\n  this.final = false\n  this.edges = {}\n  this.id = lunr.TokenSet._nextId\n  lunr.TokenSet._nextId += 1\n}\n\n/**\n * Keeps track of the next, auto increment, identifier to assign\n * to a new tokenSet.\n *\n * TokenSets require a unique identifier to be correctly minimised.\n *\n * @private\n */\nlunr.TokenSet._nextId = 1\n\n/**\n * Creates a TokenSet instance from the given sorted array of words.\n *\n * @param {String[]} arr - A sorted array of strings to create the set from.\n * @returns {lunr.TokenSet}\n * @throws Will throw an error if the input array is not sorted.\n */\nlunr.TokenSet.fromArray = function (arr) {\n  var builder = new lunr.TokenSet.Builder\n\n  for (var i = 0, len = arr.length; i < len; i++) {\n    builder.insert(arr[i])\n  }\n\n  builder.finish()\n  return builder.root\n}\n\n/**\n * Creates a token set from a query clause.\n *\n * @private\n * @param {Object} clause - A single clause from lunr.Query.\n * @param {string} clause.term - The query clause term.\n * @param {number} [clause.editDistance] - The optional edit distance for the term.\n * @returns {lunr.TokenSet}\n */\nlunr.TokenSet.fromClause = function (clause) {\n  if ('editDistance' in clause) {\n    return lunr.TokenSet.fromFuzzyString(clause.term, clause.editDistance)\n  } else {\n    return lunr.TokenSet.fromString(clause.term)\n  }\n}\n\n/**\n * Creates a token set representing a single string with a specified\n * edit distance.\n *\n * Insertions, deletions, substitutions and transpositions are each\n * treated as an edit distance of 1.\n *\n * Increasing the allowed edit distance will have a dramatic impact\n * on the performance of both creating and intersecting these TokenSets.\n * It is advised to keep the edit distance less than 3.\n *\n * @param {string} str - The string to create the token set from.\n * @param {number} editDistance - The allowed edit distance to match.\n * @returns {lunr.Vector}\n */\nlunr.TokenSet.fromFuzzyString = function (str, editDistance) {\n  var root = new lunr.TokenSet\n\n  var stack = [{\n    node: root,\n    editsRemaining: editDistance,\n    str: str\n  }]\n\n  while (stack.length) {\n    var frame = stack.pop()\n\n    // no edit\n    if (frame.str.length > 0) {\n      var char = frame.str.charAt(0),\n          noEditNode\n\n      if (char in frame.node.edges) {\n        noEditNode = frame.node.edges[char]\n      } else {\n        noEditNode = new lunr.TokenSet\n        frame.node.edges[char] = noEditNode\n      }\n\n      if (frame.str.length == 1) {\n        noEditNode.final = true\n      }\n\n      stack.push({\n        node: noEditNode,\n        editsRemaining: frame.editsRemaining,\n        str: frame.str.slice(1)\n      })\n    }\n\n    if (frame.editsRemaining == 0) {\n      continue\n    }\n\n    // insertion\n    if (\"*\" in frame.node.edges) {\n      var insertionNode = frame.node.edges[\"*\"]\n    } else {\n      var insertionNode = new lunr.TokenSet\n      frame.node.edges[\"*\"] = insertionNode\n    }\n\n    if (frame.str.length == 0) {\n      insertionNode.final = true\n    }\n\n    stack.push({\n      node: insertionNode,\n      editsRemaining: frame.editsRemaining - 1,\n      str: frame.str\n    })\n\n    // deletion\n    // can only do a deletion if we have enough edits remaining\n    // and if there are characters left to delete in the string\n    if (frame.str.length > 1) {\n      stack.push({\n        node: frame.node,\n        editsRemaining: frame.editsRemaining - 1,\n        str: frame.str.slice(1)\n      })\n    }\n\n    // deletion\n    // just removing the last character from the str\n    if (frame.str.length == 1) {\n      frame.node.final = true\n    }\n\n    // substitution\n    // can only do a substitution if we have enough edits remaining\n    // and if there are characters left to substitute\n    if (frame.str.length >= 1) {\n      if (\"*\" in frame.node.edges) {\n        var substitutionNode = frame.node.edges[\"*\"]\n      } else {\n        var substitutionNode = new lunr.TokenSet\n        frame.node.edges[\"*\"] = substitutionNode\n      }\n\n      if (frame.str.length == 1) {\n        substitutionNode.final = true\n      }\n\n      stack.push({\n        node: substitutionNode,\n        editsRemaining: frame.editsRemaining - 1,\n        str: frame.str.slice(1)\n      })\n    }\n\n    // transposition\n    // can only do a transposition if there are edits remaining\n    // and there are enough characters to transpose\n    if (frame.str.length > 1) {\n      var charA = frame.str.charAt(0),\n          charB = frame.str.charAt(1),\n          transposeNode\n\n      if (charB in frame.node.edges) {\n        transposeNode = frame.node.edges[charB]\n      } else {\n        transposeNode = new lunr.TokenSet\n        frame.node.edges[charB] = transposeNode\n      }\n\n      if (frame.str.length == 1) {\n        transposeNode.final = true\n      }\n\n      stack.push({\n        node: transposeNode,\n        editsRemaining: frame.editsRemaining - 1,\n        str: charA + frame.str.slice(2)\n      })\n    }\n  }\n\n  return root\n}\n\n/**\n * Creates a TokenSet from a string.\n *\n * The string may contain one or more wildcard characters (*)\n * that will allow wildcard matching when intersecting with\n * another TokenSet.\n *\n * @param {string} str - The string to create a TokenSet from.\n * @returns {lunr.TokenSet}\n */\nlunr.TokenSet.fromString = function (str) {\n  var node = new lunr.TokenSet,\n      root = node\n\n  /*\n   * Iterates through all characters within the passed string\n   * appending a node for each character.\n   *\n   * When a wildcard character is found then a self\n   * referencing edge is introduced to continually match\n   * any number of any characters.\n   */\n  for (var i = 0, len = str.length; i < len; i++) {\n    var char = str[i],\n        final = (i == len - 1)\n\n    if (char == \"*\") {\n      node.edges[char] = node\n      node.final = final\n\n    } else {\n      var next = new lunr.TokenSet\n      next.final = final\n\n      node.edges[char] = next\n      node = next\n    }\n  }\n\n  return root\n}\n\n/**\n * Converts this TokenSet into an array of strings\n * contained within the TokenSet.\n *\n * This is not intended to be used on a TokenSet that\n * contains wildcards, in these cases the results are\n * undefined and are likely to cause an infinite loop.\n *\n * @returns {string[]}\n */\nlunr.TokenSet.prototype.toArray = function () {\n  var words = []\n\n  var stack = [{\n    prefix: \"\",\n    node: this\n  }]\n\n  while (stack.length) {\n    var frame = stack.pop(),\n        edges = Object.keys(frame.node.edges),\n        len = edges.length\n\n    if (frame.node.final) {\n      /* In Safari, at this point the prefix is sometimes corrupted, see:\n       * https://github.com/olivernn/lunr.js/issues/279 Calling any\n       * String.prototype method forces Safari to \"cast\" this string to what\n       * it's supposed to be, fixing the bug. */\n      frame.prefix.charAt(0)\n      words.push(frame.prefix)\n    }\n\n    for (var i = 0; i < len; i++) {\n      var edge = edges[i]\n\n      stack.push({\n        prefix: frame.prefix.concat(edge),\n        node: frame.node.edges[edge]\n      })\n    }\n  }\n\n  return words\n}\n\n/**\n * Generates a string representation of a TokenSet.\n *\n * This is intended to allow TokenSets to be used as keys\n * in objects, largely to aid the construction and minimisation\n * of a TokenSet. As such it is not designed to be a human\n * friendly representation of the TokenSet.\n *\n * @returns {string}\n */\nlunr.TokenSet.prototype.toString = function () {\n  // NOTE: Using Object.keys here as this.edges is very likely\n  // to enter 'hash-mode' with many keys being added\n  //\n  // avoiding a for-in loop here as it leads to the function\n  // being de-optimised (at least in V8). From some simple\n  // benchmarks the performance is comparable, but allowing\n  // V8 to optimize may mean easy performance wins in the future.\n\n  if (this._str) {\n    return this._str\n  }\n\n  var str = this.final ? '1' : '0',\n      labels = Object.keys(this.edges).sort(),\n      len = labels.length\n\n  for (var i = 0; i < len; i++) {\n    var label = labels[i],\n        node = this.edges[label]\n\n    str = str + label + node.id\n  }\n\n  return str\n}\n\n/**\n * Returns a new TokenSet that is the intersection of\n * this TokenSet and the passed TokenSet.\n *\n * This intersection will take into account any wildcards\n * contained within the TokenSet.\n *\n * @param {lunr.TokenSet} b - An other TokenSet to intersect with.\n * @returns {lunr.TokenSet}\n */\nlunr.TokenSet.prototype.intersect = function (b) {\n  var output = new lunr.TokenSet,\n      frame = undefined\n\n  var stack = [{\n    qNode: b,\n    output: output,\n    node: this\n  }]\n\n  while (stack.length) {\n    frame = stack.pop()\n\n    // NOTE: As with the #toString method, we are using\n    // Object.keys and a for loop instead of a for-in loop\n    // as both of these objects enter 'hash' mode, causing\n    // the function to be de-optimised in V8\n    var qEdges = Object.keys(frame.qNode.edges),\n        qLen = qEdges.length,\n        nEdges = Object.keys(frame.node.edges),\n        nLen = nEdges.length\n\n    for (var q = 0; q < qLen; q++) {\n      var qEdge = qEdges[q]\n\n      for (var n = 0; n < nLen; n++) {\n        var nEdge = nEdges[n]\n\n        if (nEdge == qEdge || qEdge == '*') {\n          var node = frame.node.edges[nEdge],\n              qNode = frame.qNode.edges[qEdge],\n              final = node.final && qNode.final,\n              next = undefined\n\n          if (nEdge in frame.output.edges) {\n            // an edge already exists for this character\n            // no need to create a new node, just set the finality\n            // bit unless this node is already final\n            next = frame.output.edges[nEdge]\n            next.final = next.final || final\n\n          } else {\n            // no edge exists yet, must create one\n            // set the finality bit and insert it\n            // into the output\n            next = new lunr.TokenSet\n            next.final = final\n            frame.output.edges[nEdge] = next\n          }\n\n          stack.push({\n            qNode: qNode,\n            output: next,\n            node: node\n          })\n        }\n      }\n    }\n  }\n\n  return output\n}\nlunr.TokenSet.Builder = function () {\n  this.previousWord = \"\"\n  this.root = new lunr.TokenSet\n  this.uncheckedNodes = []\n  this.minimizedNodes = {}\n}\n\nlunr.TokenSet.Builder.prototype.insert = function (word) {\n  var node,\n      commonPrefix = 0\n\n  if (word < this.previousWord) {\n    throw new Error (\"Out of order word insertion\")\n  }\n\n  for (var i = 0; i < word.length && i < this.previousWord.length; i++) {\n    if (word[i] != this.previousWord[i]) break\n    commonPrefix++\n  }\n\n  this.minimize(commonPrefix)\n\n  if (this.uncheckedNodes.length == 0) {\n    node = this.root\n  } else {\n    node = this.uncheckedNodes[this.uncheckedNodes.length - 1].child\n  }\n\n  for (var i = commonPrefix; i < word.length; i++) {\n    var nextNode = new lunr.TokenSet,\n        char = word[i]\n\n    node.edges[char] = nextNode\n\n    this.uncheckedNodes.push({\n      parent: node,\n      char: char,\n      child: nextNode\n    })\n\n    node = nextNode\n  }\n\n  node.final = true\n  this.previousWord = word\n}\n\nlunr.TokenSet.Builder.prototype.finish = function () {\n  this.minimize(0)\n}\n\nlunr.TokenSet.Builder.prototype.minimize = function (downTo) {\n  for (var i = this.uncheckedNodes.length - 1; i >= downTo; i--) {\n    var node = this.uncheckedNodes[i],\n        childKey = node.child.toString()\n\n    if (childKey in this.minimizedNodes) {\n      node.parent.edges[node.char] = this.minimizedNodes[childKey]\n    } else {\n      // Cache the key for this node since\n      // we know it can't change anymore\n      node.child._str = childKey\n\n      this.minimizedNodes[childKey] = node.child\n    }\n\n    this.uncheckedNodes.pop()\n  }\n}\n/*!\n * lunr.Index\n * Copyright (C) 2020 Oliver Nightingale\n */\n\n/**\n * An index contains the built index of all documents and provides a query interface\n * to the index.\n *\n * Usually instances of lunr.Index will not be created using this constructor, instead\n * lunr.Builder should be used to construct new indexes, or lunr.Index.load should be\n * used to load previously built and serialized indexes.\n *\n * @constructor\n * @param {Object} attrs - The attributes of the built search index.\n * @param {Object} attrs.invertedIndex - An index of term/field to document reference.\n * @param {Object<string, lunr.Vector>} attrs.fieldVectors - Field vectors\n * @param {lunr.TokenSet} attrs.tokenSet - An set of all corpus tokens.\n * @param {string[]} attrs.fields - The names of indexed document fields.\n * @param {lunr.Pipeline} attrs.pipeline - The pipeline to use for search terms.\n */\nlunr.Index = function (attrs) {\n  this.invertedIndex = attrs.invertedIndex\n  this.fieldVectors = attrs.fieldVectors\n  this.tokenSet = attrs.tokenSet\n  this.fields = attrs.fields\n  this.pipeline = attrs.pipeline\n}\n\n/**\n * A result contains details of a document matching a search query.\n * @typedef {Object} lunr.Index~Result\n * @property {string} ref - The reference of the document this result represents.\n * @property {number} score - A number between 0 and 1 representing how similar this document is to the query.\n * @property {lunr.MatchData} matchData - Contains metadata about this match including which term(s) caused the match.\n */\n\n/**\n * Although lunr provides the ability to create queries using lunr.Query, it also provides a simple\n * query language which itself is parsed into an instance of lunr.Query.\n *\n * For programmatically building queries it is advised to directly use lunr.Query, the query language\n * is best used for human entered text rather than program generated text.\n *\n * At its simplest queries can just be a single term, e.g. `hello`, multiple terms are also supported\n * and will be combined with OR, e.g `hello world` will match documents that contain either 'hello'\n * or 'world', though those that contain both will rank higher in the results.\n *\n * Wildcards can be included in terms to match one or more unspecified characters, these wildcards can\n * be inserted anywhere within the term, and more than one wildcard can exist in a single term. Adding\n * wildcards will increase the number of documents that will be found but can also have a negative\n * impact on query performance, especially with wildcards at the beginning of a term.\n *\n * Terms can be restricted to specific fields, e.g. `title:hello`, only documents with the term\n * hello in the title field will match this query. Using a field not present in the index will lead\n * to an error being thrown.\n *\n * Modifiers can also be added to terms, lunr supports edit distance and boost modifiers on terms. A term\n * boost will make documents matching that term score higher, e.g. `foo^5`. Edit distance is also supported\n * to provide fuzzy matching, e.g. 'hello~2' will match documents with hello with an edit distance of 2.\n * Avoid large values for edit distance to improve query performance.\n *\n * Each term also supports a presence modifier. By default a term's presence in document is optional, however\n * this can be changed to either required or prohibited. For a term's presence to be required in a document the\n * term should be prefixed with a '+', e.g. `+foo bar` is a search for documents that must contain 'foo' and\n * optionally contain 'bar'. Conversely a leading '-' sets the terms presence to prohibited, i.e. it must not\n * appear in a document, e.g. `-foo bar` is a search for documents that do not contain 'foo' but may contain 'bar'.\n *\n * To escape special characters the backslash character '\\' can be used, this allows searches to include\n * characters that would normally be considered modifiers, e.g. `foo\\~2` will search for a term \"foo~2\" instead\n * of attempting to apply a boost of 2 to the search term \"foo\".\n *\n * @typedef {string} lunr.Index~QueryString\n * @example <caption>Simple single term query</caption>\n * hello\n * @example <caption>Multiple term query</caption>\n * hello world\n * @example <caption>term scoped to a field</caption>\n * title:hello\n * @example <caption>term with a boost of 10</caption>\n * hello^10\n * @example <caption>term with an edit distance of 2</caption>\n * hello~2\n * @example <caption>terms with presence modifiers</caption>\n * -foo +bar baz\n */\n\n/**\n * Performs a search against the index using lunr query syntax.\n *\n * Results will be returned sorted by their score, the most relevant results\n * will be returned first.  For details on how the score is calculated, please see\n * the {@link https://lunrjs.com/guides/searching.html#scoring|guide}.\n *\n * For more programmatic querying use lunr.Index#query.\n *\n * @param {lunr.Index~QueryString} queryString - A string containing a lunr query.\n * @throws {lunr.QueryParseError} If the passed query string cannot be parsed.\n * @returns {lunr.Index~Result[]}\n */\nlunr.Index.prototype.search = function (queryString) {\n  return this.query(function (query) {\n    var parser = new lunr.QueryParser(queryString, query)\n    parser.parse()\n  })\n}\n\n/**\n * A query builder callback provides a query object to be used to express\n * the query to perform on the index.\n *\n * @callback lunr.Index~queryBuilder\n * @param {lunr.Query} query - The query object to build up.\n * @this lunr.Query\n */\n\n/**\n * Performs a query against the index using the yielded lunr.Query object.\n *\n * If performing programmatic queries against the index, this method is preferred\n * over lunr.Index#search so as to avoid the additional query parsing overhead.\n *\n * A query object is yielded to the supplied function which should be used to\n * express the query to be run against the index.\n *\n * Note that although this function takes a callback parameter it is _not_ an\n * asynchronous operation, the callback is just yielded a query object to be\n * customized.\n *\n * @param {lunr.Index~queryBuilder} fn - A function that is used to build the query.\n * @returns {lunr.Index~Result[]}\n */\nlunr.Index.prototype.query = function (fn) {\n  // for each query clause\n  // * process terms\n  // * expand terms from token set\n  // * find matching documents and metadata\n  // * get document vectors\n  // * score documents\n\n  var query = new lunr.Query(this.fields),\n      matchingFields = Object.create(null),\n      queryVectors = Object.create(null),\n      termFieldCache = Object.create(null),\n      requiredMatches = Object.create(null),\n      prohibitedMatches = Object.create(null)\n\n  /*\n   * To support field level boosts a query vector is created per\n   * field. An empty vector is eagerly created to support negated\n   * queries.\n   */\n  for (var i = 0; i < this.fields.length; i++) {\n    queryVectors[this.fields[i]] = new lunr.Vector\n  }\n\n  fn.call(query, query)\n\n  for (var i = 0; i < query.clauses.length; i++) {\n    /*\n     * Unless the pipeline has been disabled for this term, which is\n     * the case for terms with wildcards, we need to pass the clause\n     * term through the search pipeline. A pipeline returns an array\n     * of processed terms. Pipeline functions may expand the passed\n     * term, which means we may end up performing multiple index lookups\n     * for a single query term.\n     */\n    var clause = query.clauses[i],\n        terms = null,\n        clauseMatches = lunr.Set.empty\n\n    if (clause.usePipeline) {\n      terms = this.pipeline.runString(clause.term, {\n        fields: clause.fields\n      })\n    } else {\n      terms = [clause.term]\n    }\n\n    for (var m = 0; m < terms.length; m++) {\n      var term = terms[m]\n\n      /*\n       * Each term returned from the pipeline needs to use the same query\n       * clause object, e.g. the same boost and or edit distance. The\n       * simplest way to do this is to re-use the clause object but mutate\n       * its term property.\n       */\n      clause.term = term\n\n      /*\n       * From the term in the clause we create a token set which will then\n       * be used to intersect the indexes token set to get a list of terms\n       * to lookup in the inverted index\n       */\n      var termTokenSet = lunr.TokenSet.fromClause(clause),\n          expandedTerms = this.tokenSet.intersect(termTokenSet).toArray()\n\n      /*\n       * If a term marked as required does not exist in the tokenSet it is\n       * impossible for the search to return any matches. We set all the field\n       * scoped required matches set to empty and stop examining any further\n       * clauses.\n       */\n      if (expandedTerms.length === 0 && clause.presence === lunr.Query.presence.REQUIRED) {\n        for (var k = 0; k < clause.fields.length; k++) {\n          var field = clause.fields[k]\n          requiredMatches[field] = lunr.Set.empty\n        }\n\n        break\n      }\n\n      for (var j = 0; j < expandedTerms.length; j++) {\n        /*\n         * For each term get the posting and termIndex, this is required for\n         * building the query vector.\n         */\n        var expandedTerm = expandedTerms[j],\n            posting = this.invertedIndex[expandedTerm],\n            termIndex = posting._index\n\n        for (var k = 0; k < clause.fields.length; k++) {\n          /*\n           * For each field that this query term is scoped by (by default\n           * all fields are in scope) we need to get all the document refs\n           * that have this term in that field.\n           *\n           * The posting is the entry in the invertedIndex for the matching\n           * term from above.\n           */\n          var field = clause.fields[k],\n              fieldPosting = posting[field],\n              matchingDocumentRefs = Object.keys(fieldPosting),\n              termField = expandedTerm + \"/\" + field,\n              matchingDocumentsSet = new lunr.Set(matchingDocumentRefs)\n\n          /*\n           * if the presence of this term is required ensure that the matching\n           * documents are added to the set of required matches for this clause.\n           *\n           */\n          if (clause.presence == lunr.Query.presence.REQUIRED) {\n            clauseMatches = clauseMatches.union(matchingDocumentsSet)\n\n            if (requiredMatches[field] === undefined) {\n              requiredMatches[field] = lunr.Set.complete\n            }\n          }\n\n          /*\n           * if the presence of this term is prohibited ensure that the matching\n           * documents are added to the set of prohibited matches for this field,\n           * creating that set if it does not yet exist.\n           */\n          if (clause.presence == lunr.Query.presence.PROHIBITED) {\n            if (prohibitedMatches[field] === undefined) {\n              prohibitedMatches[field] = lunr.Set.empty\n            }\n\n            prohibitedMatches[field] = prohibitedMatches[field].union(matchingDocumentsSet)\n\n            /*\n             * Prohibited matches should not be part of the query vector used for\n             * similarity scoring and no metadata should be extracted so we continue\n             * to the next field\n             */\n            continue\n          }\n\n          /*\n           * The query field vector is populated using the termIndex found for\n           * the term and a unit value with the appropriate boost applied.\n           * Using upsert because there could already be an entry in the vector\n           * for the term we are working with. In that case we just add the scores\n           * together.\n           */\n          queryVectors[field].upsert(termIndex, clause.boost, function (a, b) { return a + b })\n\n          /**\n           * If we've already seen this term, field combo then we've already collected\n           * the matching documents and metadata, no need to go through all that again\n           */\n          if (termFieldCache[termField]) {\n            continue\n          }\n\n          for (var l = 0; l < matchingDocumentRefs.length; l++) {\n            /*\n             * All metadata for this term/field/document triple\n             * are then extracted and collected into an instance\n             * of lunr.MatchData ready to be returned in the query\n             * results\n             */\n            var matchingDocumentRef = matchingDocumentRefs[l],\n                matchingFieldRef = new lunr.FieldRef (matchingDocumentRef, field),\n                metadata = fieldPosting[matchingDocumentRef],\n                fieldMatch\n\n            if ((fieldMatch = matchingFields[matchingFieldRef]) === undefined) {\n              matchingFields[matchingFieldRef] = new lunr.MatchData (expandedTerm, field, metadata)\n            } else {\n              fieldMatch.add(expandedTerm, field, metadata)\n            }\n\n          }\n\n          termFieldCache[termField] = true\n        }\n      }\n    }\n\n    /**\n     * If the presence was required we need to update the requiredMatches field sets.\n     * We do this after all fields for the term have collected their matches because\n     * the clause terms presence is required in _any_ of the fields not _all_ of the\n     * fields.\n     */\n    if (clause.presence === lunr.Query.presence.REQUIRED) {\n      for (var k = 0; k < clause.fields.length; k++) {\n        var field = clause.fields[k]\n        requiredMatches[field] = requiredMatches[field].intersect(clauseMatches)\n      }\n    }\n  }\n\n  /**\n   * Need to combine the field scoped required and prohibited\n   * matching documents into a global set of required and prohibited\n   * matches\n   */\n  var allRequiredMatches = lunr.Set.complete,\n      allProhibitedMatches = lunr.Set.empty\n\n  for (var i = 0; i < this.fields.length; i++) {\n    var field = this.fields[i]\n\n    if (requiredMatches[field]) {\n      allRequiredMatches = allRequiredMatches.intersect(requiredMatches[field])\n    }\n\n    if (prohibitedMatches[field]) {\n      allProhibitedMatches = allProhibitedMatches.union(prohibitedMatches[field])\n    }\n  }\n\n  var matchingFieldRefs = Object.keys(matchingFields),\n      results = [],\n      matches = Object.create(null)\n\n  /*\n   * If the query is negated (contains only prohibited terms)\n   * we need to get _all_ fieldRefs currently existing in the\n   * index. This is only done when we know that the query is\n   * entirely prohibited terms to avoid any cost of getting all\n   * fieldRefs unnecessarily.\n   *\n   * Additionally, blank MatchData must be created to correctly\n   * populate the results.\n   */\n  if (query.isNegated()) {\n    matchingFieldRefs = Object.keys(this.fieldVectors)\n\n    for (var i = 0; i < matchingFieldRefs.length; i++) {\n      var matchingFieldRef = matchingFieldRefs[i]\n      var fieldRef = lunr.FieldRef.fromString(matchingFieldRef)\n      matchingFields[matchingFieldRef] = new lunr.MatchData\n    }\n  }\n\n  for (var i = 0; i < matchingFieldRefs.length; i++) {\n    /*\n     * Currently we have document fields that match the query, but we\n     * need to return documents. The matchData and scores are combined\n     * from multiple fields belonging to the same document.\n     *\n     * Scores are calculated by field, using the query vectors created\n     * above, and combined into a final document score using addition.\n     */\n    var fieldRef = lunr.FieldRef.fromString(matchingFieldRefs[i]),\n        docRef = fieldRef.docRef\n\n    if (!allRequiredMatches.contains(docRef)) {\n      continue\n    }\n\n    if (allProhibitedMatches.contains(docRef)) {\n      continue\n    }\n\n    var fieldVector = this.fieldVectors[fieldRef],\n        score = queryVectors[fieldRef.fieldName].similarity(fieldVector),\n        docMatch\n\n    if ((docMatch = matches[docRef]) !== undefined) {\n      docMatch.score += score\n      docMatch.matchData.combine(matchingFields[fieldRef])\n    } else {\n      var match = {\n        ref: docRef,\n        score: score,\n        matchData: matchingFields[fieldRef]\n      }\n      matches[docRef] = match\n      results.push(match)\n    }\n  }\n\n  /*\n   * Sort the results objects by score, highest first.\n   */\n  return results.sort(function (a, b) {\n    return b.score - a.score\n  })\n}\n\n/**\n * Prepares the index for JSON serialization.\n *\n * The schema for this JSON blob will be described in a\n * separate JSON schema file.\n *\n * @returns {Object}\n */\nlunr.Index.prototype.toJSON = function () {\n  var invertedIndex = Object.keys(this.invertedIndex)\n    .sort()\n    .map(function (term) {\n      return [term, this.invertedIndex[term]]\n    }, this)\n\n  var fieldVectors = Object.keys(this.fieldVectors)\n    .map(function (ref) {\n      return [ref, this.fieldVectors[ref].toJSON()]\n    }, this)\n\n  return {\n    version: lunr.version,\n    fields: this.fields,\n    fieldVectors: fieldVectors,\n    invertedIndex: invertedIndex,\n    pipeline: this.pipeline.toJSON()\n  }\n}\n\n/**\n * Loads a previously serialized lunr.Index\n *\n * @param {Object} serializedIndex - A previously serialized lunr.Index\n * @returns {lunr.Index}\n */\nlunr.Index.load = function (serializedIndex) {\n  var attrs = {},\n      fieldVectors = {},\n      serializedVectors = serializedIndex.fieldVectors,\n      invertedIndex = Object.create(null),\n      serializedInvertedIndex = serializedIndex.invertedIndex,\n      tokenSetBuilder = new lunr.TokenSet.Builder,\n      pipeline = lunr.Pipeline.load(serializedIndex.pipeline)\n\n  if (serializedIndex.version != lunr.version) {\n    lunr.utils.warn(\"Version mismatch when loading serialised index. Current version of lunr '\" + lunr.version + \"' does not match serialized index '\" + serializedIndex.version + \"'\")\n  }\n\n  for (var i = 0; i < serializedVectors.length; i++) {\n    var tuple = serializedVectors[i],\n        ref = tuple[0],\n        elements = tuple[1]\n\n    fieldVectors[ref] = new lunr.Vector(elements)\n  }\n\n  for (var i = 0; i < serializedInvertedIndex.length; i++) {\n    var tuple = serializedInvertedIndex[i],\n        term = tuple[0],\n        posting = tuple[1]\n\n    tokenSetBuilder.insert(term)\n    invertedIndex[term] = posting\n  }\n\n  tokenSetBuilder.finish()\n\n  attrs.fields = serializedIndex.fields\n\n  attrs.fieldVectors = fieldVectors\n  attrs.invertedIndex = invertedIndex\n  attrs.tokenSet = tokenSetBuilder.root\n  attrs.pipeline = pipeline\n\n  return new lunr.Index(attrs)\n}\n/*!\n * lunr.Builder\n * Copyright (C) 2020 Oliver Nightingale\n */\n\n/**\n * lunr.Builder performs indexing on a set of documents and\n * returns instances of lunr.Index ready for querying.\n *\n * All configuration of the index is done via the builder, the\n * fields to index, the document reference, the text processing\n * pipeline and document scoring parameters are all set on the\n * builder before indexing.\n *\n * @constructor\n * @property {string} _ref - Internal reference to the document reference field.\n * @property {string[]} _fields - Internal reference to the document fields to index.\n * @property {object} invertedIndex - The inverted index maps terms to document fields.\n * @property {object} documentTermFrequencies - Keeps track of document term frequencies.\n * @property {object} documentLengths - Keeps track of the length of documents added to the index.\n * @property {lunr.tokenizer} tokenizer - Function for splitting strings into tokens for indexing.\n * @property {lunr.Pipeline} pipeline - The pipeline performs text processing on tokens before indexing.\n * @property {lunr.Pipeline} searchPipeline - A pipeline for processing search terms before querying the index.\n * @property {number} documentCount - Keeps track of the total number of documents indexed.\n * @property {number} _b - A parameter to control field length normalization, setting this to 0 disabled normalization, 1 fully normalizes field lengths, the default value is 0.75.\n * @property {number} _k1 - A parameter to control how quickly an increase in term frequency results in term frequency saturation, the default value is 1.2.\n * @property {number} termIndex - A counter incremented for each unique term, used to identify a terms position in the vector space.\n * @property {array} metadataWhitelist - A list of metadata keys that have been whitelisted for entry in the index.\n */\nlunr.Builder = function () {\n  this._ref = \"id\"\n  this._fields = Object.create(null)\n  this._documents = Object.create(null)\n  this.invertedIndex = Object.create(null)\n  this.fieldTermFrequencies = {}\n  this.fieldLengths = {}\n  this.tokenizer = lunr.tokenizer\n  this.pipeline = new lunr.Pipeline\n  this.searchPipeline = new lunr.Pipeline\n  this.documentCount = 0\n  this._b = 0.75\n  this._k1 = 1.2\n  this.termIndex = 0\n  this.metadataWhitelist = []\n}\n\n/**\n * Sets the document field used as the document reference. Every document must have this field.\n * The type of this field in the document should be a string, if it is not a string it will be\n * coerced into a string by calling toString.\n *\n * The default ref is 'id'.\n *\n * The ref should _not_ be changed during indexing, it should be set before any documents are\n * added to the index. Changing it during indexing can lead to inconsistent results.\n *\n * @param {string} ref - The name of the reference field in the document.\n */\nlunr.Builder.prototype.ref = function (ref) {\n  this._ref = ref\n}\n\n/**\n * A function that is used to extract a field from a document.\n *\n * Lunr expects a field to be at the top level of a document, if however the field\n * is deeply nested within a document an extractor function can be used to extract\n * the right field for indexing.\n *\n * @callback fieldExtractor\n * @param {object} doc - The document being added to the index.\n * @returns {?(string|object|object[])} obj - The object that will be indexed for this field.\n * @example <caption>Extracting a nested field</caption>\n * function (doc) { return doc.nested.field }\n */\n\n/**\n * Adds a field to the list of document fields that will be indexed. Every document being\n * indexed should have this field. Null values for this field in indexed documents will\n * not cause errors but will limit the chance of that document being retrieved by searches.\n *\n * All fields should be added before adding documents to the index. Adding fields after\n * a document has been indexed will have no effect on already indexed documents.\n *\n * Fields can be boosted at build time. This allows terms within that field to have more\n * importance when ranking search results. Use a field boost to specify that matches within\n * one field are more important than other fields.\n *\n * @param {string} fieldName - The name of a field to index in all documents.\n * @param {object} attributes - Optional attributes associated with this field.\n * @param {number} [attributes.boost=1] - Boost applied to all terms within this field.\n * @param {fieldExtractor} [attributes.extractor] - Function to extract a field from a document.\n * @throws {RangeError} fieldName cannot contain unsupported characters '/'\n */\nlunr.Builder.prototype.field = function (fieldName, attributes) {\n  if (/\\//.test(fieldName)) {\n    throw new RangeError (\"Field '\" + fieldName + \"' contains illegal character '/'\")\n  }\n\n  this._fields[fieldName] = attributes || {}\n}\n\n/**\n * A parameter to tune the amount of field length normalisation that is applied when\n * calculating relevance scores. A value of 0 will completely disable any normalisation\n * and a value of 1 will fully normalise field lengths. The default is 0.75. Values of b\n * will be clamped to the range 0 - 1.\n *\n * @param {number} number - The value to set for this tuning parameter.\n */\nlunr.Builder.prototype.b = function (number) {\n  if (number < 0) {\n    this._b = 0\n  } else if (number > 1) {\n    this._b = 1\n  } else {\n    this._b = number\n  }\n}\n\n/**\n * A parameter that controls the speed at which a rise in term frequency results in term\n * frequency saturation. The default value is 1.2. Setting this to a higher value will give\n * slower saturation levels, a lower value will result in quicker saturation.\n *\n * @param {number} number - The value to set for this tuning parameter.\n */\nlunr.Builder.prototype.k1 = function (number) {\n  this._k1 = number\n}\n\n/**\n * Adds a document to the index.\n *\n * Before adding fields to the index the index should have been fully setup, with the document\n * ref and all fields to index already having been specified.\n *\n * The document must have a field name as specified by the ref (by default this is 'id') and\n * it should have all fields defined for indexing, though null or undefined values will not\n * cause errors.\n *\n * Entire documents can be boosted at build time. Applying a boost to a document indicates that\n * this document should rank higher in search results than other documents.\n *\n * @param {object} doc - The document to add to the index.\n * @param {object} attributes - Optional attributes associated with this document.\n * @param {number} [attributes.boost=1] - Boost applied to all terms within this document.\n */\nlunr.Builder.prototype.add = function (doc, attributes) {\n  var docRef = doc[this._ref],\n      fields = Object.keys(this._fields)\n\n  this._documents[docRef] = attributes || {}\n  this.documentCount += 1\n\n  for (var i = 0; i < fields.length; i++) {\n    var fieldName = fields[i],\n        extractor = this._fields[fieldName].extractor,\n        field = extractor ? extractor(doc) : doc[fieldName],\n        tokens = this.tokenizer(field, {\n          fields: [fieldName]\n        }),\n        terms = this.pipeline.run(tokens),\n        fieldRef = new lunr.FieldRef (docRef, fieldName),\n        fieldTerms = Object.create(null)\n\n    this.fieldTermFrequencies[fieldRef] = fieldTerms\n    this.fieldLengths[fieldRef] = 0\n\n    // store the length of this field for this document\n    this.fieldLengths[fieldRef] += terms.length\n\n    // calculate term frequencies for this field\n    for (var j = 0; j < terms.length; j++) {\n      var term = terms[j]\n\n      if (fieldTerms[term] == undefined) {\n        fieldTerms[term] = 0\n      }\n\n      fieldTerms[term] += 1\n\n      // add to inverted index\n      // create an initial posting if one doesn't exist\n      if (this.invertedIndex[term] == undefined) {\n        var posting = Object.create(null)\n        posting[\"_index\"] = this.termIndex\n        this.termIndex += 1\n\n        for (var k = 0; k < fields.length; k++) {\n          posting[fields[k]] = Object.create(null)\n        }\n\n        this.invertedIndex[term] = posting\n      }\n\n      // add an entry for this term/fieldName/docRef to the invertedIndex\n      if (this.invertedIndex[term][fieldName][docRef] == undefined) {\n        this.invertedIndex[term][fieldName][docRef] = Object.create(null)\n      }\n\n      // store all whitelisted metadata about this token in the\n      // inverted index\n      for (var l = 0; l < this.metadataWhitelist.length; l++) {\n        var metadataKey = this.metadataWhitelist[l],\n            metadata = term.metadata[metadataKey]\n\n        if (this.invertedIndex[term][fieldName][docRef][metadataKey] == undefined) {\n          this.invertedIndex[term][fieldName][docRef][metadataKey] = []\n        }\n\n        this.invertedIndex[term][fieldName][docRef][metadataKey].push(metadata)\n      }\n    }\n\n  }\n}\n\n/**\n * Calculates the average document length for this index\n *\n * @private\n */\nlunr.Builder.prototype.calculateAverageFieldLengths = function () {\n\n  var fieldRefs = Object.keys(this.fieldLengths),\n      numberOfFields = fieldRefs.length,\n      accumulator = {},\n      documentsWithField = {}\n\n  for (var i = 0; i < numberOfFields; i++) {\n    var fieldRef = lunr.FieldRef.fromString(fieldRefs[i]),\n        field = fieldRef.fieldName\n\n    documentsWithField[field] || (documentsWithField[field] = 0)\n    documentsWithField[field] += 1\n\n    accumulator[field] || (accumulator[field] = 0)\n    accumulator[field] += this.fieldLengths[fieldRef]\n  }\n\n  var fields = Object.keys(this._fields)\n\n  for (var i = 0; i < fields.length; i++) {\n    var fieldName = fields[i]\n    accumulator[fieldName] = accumulator[fieldName] / documentsWithField[fieldName]\n  }\n\n  this.averageFieldLength = accumulator\n}\n\n/**\n * Builds a vector space model of every document using lunr.Vector\n *\n * @private\n */\nlunr.Builder.prototype.createFieldVectors = function () {\n  var fieldVectors = {},\n      fieldRefs = Object.keys(this.fieldTermFrequencies),\n      fieldRefsLength = fieldRefs.length,\n      termIdfCache = Object.create(null)\n\n  for (var i = 0; i < fieldRefsLength; i++) {\n    var fieldRef = lunr.FieldRef.fromString(fieldRefs[i]),\n        fieldName = fieldRef.fieldName,\n        fieldLength = this.fieldLengths[fieldRef],\n        fieldVector = new lunr.Vector,\n        termFrequencies = this.fieldTermFrequencies[fieldRef],\n        terms = Object.keys(termFrequencies),\n        termsLength = terms.length\n\n\n    var fieldBoost = this._fields[fieldName].boost || 1,\n        docBoost = this._documents[fieldRef.docRef].boost || 1\n\n    for (var j = 0; j < termsLength; j++) {\n      var term = terms[j],\n          tf = termFrequencies[term],\n          termIndex = this.invertedIndex[term]._index,\n          idf, score, scoreWithPrecision\n\n      if (termIdfCache[term] === undefined) {\n        idf = lunr.idf(this.invertedIndex[term], this.documentCount)\n        termIdfCache[term] = idf\n      } else {\n        idf = termIdfCache[term]\n      }\n\n      score = idf * ((this._k1 + 1) * tf) / (this._k1 * (1 - this._b + this._b * (fieldLength / this.averageFieldLength[fieldName])) + tf)\n      score *= fieldBoost\n      score *= docBoost\n      scoreWithPrecision = Math.round(score * 1000) / 1000\n      // Converts 1.23456789 to 1.234.\n      // Reducing the precision so that the vectors take up less\n      // space when serialised. Doing it now so that they behave\n      // the same before and after serialisation. Also, this is\n      // the fastest approach to reducing a number's precision in\n      // JavaScript.\n\n      fieldVector.insert(termIndex, scoreWithPrecision)\n    }\n\n    fieldVectors[fieldRef] = fieldVector\n  }\n\n  this.fieldVectors = fieldVectors\n}\n\n/**\n * Creates a token set of all tokens in the index using lunr.TokenSet\n *\n * @private\n */\nlunr.Builder.prototype.createTokenSet = function () {\n  this.tokenSet = lunr.TokenSet.fromArray(\n    Object.keys(this.invertedIndex).sort()\n  )\n}\n\n/**\n * Builds the index, creating an instance of lunr.Index.\n *\n * This completes the indexing process and should only be called\n * once all documents have been added to the index.\n *\n * @returns {lunr.Index}\n */\nlunr.Builder.prototype.build = function () {\n  this.calculateAverageFieldLengths()\n  this.createFieldVectors()\n  this.createTokenSet()\n\n  return new lunr.Index({\n    invertedIndex: this.invertedIndex,\n    fieldVectors: this.fieldVectors,\n    tokenSet: this.tokenSet,\n    fields: Object.keys(this._fields),\n    pipeline: this.searchPipeline\n  })\n}\n\n/**\n * Applies a plugin to the index builder.\n *\n * A plugin is a function that is called with the index builder as its context.\n * Plugins can be used to customise or extend the behaviour of the index\n * in some way. A plugin is just a function, that encapsulated the custom\n * behaviour that should be applied when building the index.\n *\n * The plugin function will be called with the index builder as its argument, additional\n * arguments can also be passed when calling use. The function will be called\n * with the index builder as its context.\n *\n * @param {Function} plugin The plugin to apply.\n */\nlunr.Builder.prototype.use = function (fn) {\n  var args = Array.prototype.slice.call(arguments, 1)\n  args.unshift(this)\n  fn.apply(this, args)\n}\n/**\n * Contains and collects metadata about a matching document.\n * A single instance of lunr.MatchData is returned as part of every\n * lunr.Index~Result.\n *\n * @constructor\n * @param {string} term - The term this match data is associated with\n * @param {string} field - The field in which the term was found\n * @param {object} metadata - The metadata recorded about this term in this field\n * @property {object} metadata - A cloned collection of metadata associated with this document.\n * @see {@link lunr.Index~Result}\n */\nlunr.MatchData = function (term, field, metadata) {\n  var clonedMetadata = Object.create(null),\n      metadataKeys = Object.keys(metadata || {})\n\n  // Cloning the metadata to prevent the original\n  // being mutated during match data combination.\n  // Metadata is kept in an array within the inverted\n  // index so cloning the data can be done with\n  // Array#slice\n  for (var i = 0; i < metadataKeys.length; i++) {\n    var key = metadataKeys[i]\n    clonedMetadata[key] = metadata[key].slice()\n  }\n\n  this.metadata = Object.create(null)\n\n  if (term !== undefined) {\n    this.metadata[term] = Object.create(null)\n    this.metadata[term][field] = clonedMetadata\n  }\n}\n\n/**\n * An instance of lunr.MatchData will be created for every term that matches a\n * document. However only one instance is required in a lunr.Index~Result. This\n * method combines metadata from another instance of lunr.MatchData with this\n * objects metadata.\n *\n * @param {lunr.MatchData} otherMatchData - Another instance of match data to merge with this one.\n * @see {@link lunr.Index~Result}\n */\nlunr.MatchData.prototype.combine = function (otherMatchData) {\n  var terms = Object.keys(otherMatchData.metadata)\n\n  for (var i = 0; i < terms.length; i++) {\n    var term = terms[i],\n        fields = Object.keys(otherMatchData.metadata[term])\n\n    if (this.metadata[term] == undefined) {\n      this.metadata[term] = Object.create(null)\n    }\n\n    for (var j = 0; j < fields.length; j++) {\n      var field = fields[j],\n          keys = Object.keys(otherMatchData.metadata[term][field])\n\n      if (this.metadata[term][field] == undefined) {\n        this.metadata[term][field] = Object.create(null)\n      }\n\n      for (var k = 0; k < keys.length; k++) {\n        var key = keys[k]\n\n        if (this.metadata[term][field][key] == undefined) {\n          this.metadata[term][field][key] = otherMatchData.metadata[term][field][key]\n        } else {\n          this.metadata[term][field][key] = this.metadata[term][field][key].concat(otherMatchData.metadata[term][field][key])\n        }\n\n      }\n    }\n  }\n}\n\n/**\n * Add metadata for a term/field pair to this instance of match data.\n *\n * @param {string} term - The term this match data is associated with\n * @param {string} field - The field in which the term was found\n * @param {object} metadata - The metadata recorded about this term in this field\n */\nlunr.MatchData.prototype.add = function (term, field, metadata) {\n  if (!(term in this.metadata)) {\n    this.metadata[term] = Object.create(null)\n    this.metadata[term][field] = metadata\n    return\n  }\n\n  if (!(field in this.metadata[term])) {\n    this.metadata[term][field] = metadata\n    return\n  }\n\n  var metadataKeys = Object.keys(metadata)\n\n  for (var i = 0; i < metadataKeys.length; i++) {\n    var key = metadataKeys[i]\n\n    if (key in this.metadata[term][field]) {\n      this.metadata[term][field][key] = this.metadata[term][field][key].concat(metadata[key])\n    } else {\n      this.metadata[term][field][key] = metadata[key]\n    }\n  }\n}\n/**\n * A lunr.Query provides a programmatic way of defining queries to be performed\n * against a {@link lunr.Index}.\n *\n * Prefer constructing a lunr.Query using the {@link lunr.Index#query} method\n * so the query object is pre-initialized with the right index fields.\n *\n * @constructor\n * @property {lunr.Query~Clause[]} clauses - An array of query clauses.\n * @property {string[]} allFields - An array of all available fields in a lunr.Index.\n */\nlunr.Query = function (allFields) {\n  this.clauses = []\n  this.allFields = allFields\n}\n\n/**\n * Constants for indicating what kind of automatic wildcard insertion will be used when constructing a query clause.\n *\n * This allows wildcards to be added to the beginning and end of a term without having to manually do any string\n * concatenation.\n *\n * The wildcard constants can be bitwise combined to select both leading and trailing wildcards.\n *\n * @constant\n * @default\n * @property {number} wildcard.NONE - The term will have no wildcards inserted, this is the default behaviour\n * @property {number} wildcard.LEADING - Prepend the term with a wildcard, unless a leading wildcard already exists\n * @property {number} wildcard.TRAILING - Append a wildcard to the term, unless a trailing wildcard already exists\n * @see lunr.Query~Clause\n * @see lunr.Query#clause\n * @see lunr.Query#term\n * @example <caption>query term with trailing wildcard</caption>\n * query.term('foo', { wildcard: lunr.Query.wildcard.TRAILING })\n * @example <caption>query term with leading and trailing wildcard</caption>\n * query.term('foo', {\n *   wildcard: lunr.Query.wildcard.LEADING | lunr.Query.wildcard.TRAILING\n * })\n */\n\nlunr.Query.wildcard = new String (\"*\")\nlunr.Query.wildcard.NONE = 0\nlunr.Query.wildcard.LEADING = 1\nlunr.Query.wildcard.TRAILING = 2\n\n/**\n * Constants for indicating what kind of presence a term must have in matching documents.\n *\n * @constant\n * @enum {number}\n * @see lunr.Query~Clause\n * @see lunr.Query#clause\n * @see lunr.Query#term\n * @example <caption>query term with required presence</caption>\n * query.term('foo', { presence: lunr.Query.presence.REQUIRED })\n */\nlunr.Query.presence = {\n  /**\n   * Term's presence in a document is optional, this is the default value.\n   */\n  OPTIONAL: 1,\n\n  /**\n   * Term's presence in a document is required, documents that do not contain\n   * this term will not be returned.\n   */\n  REQUIRED: 2,\n\n  /**\n   * Term's presence in a document is prohibited, documents that do contain\n   * this term will not be returned.\n   */\n  PROHIBITED: 3\n}\n\n/**\n * A single clause in a {@link lunr.Query} contains a term and details on how to\n * match that term against a {@link lunr.Index}.\n *\n * @typedef {Object} lunr.Query~Clause\n * @property {string[]} fields - The fields in an index this clause should be matched against.\n * @property {number} [boost=1] - Any boost that should be applied when matching this clause.\n * @property {number} [editDistance] - Whether the term should have fuzzy matching applied, and how fuzzy the match should be.\n * @property {boolean} [usePipeline] - Whether the term should be passed through the search pipeline.\n * @property {number} [wildcard=lunr.Query.wildcard.NONE] - Whether the term should have wildcards appended or prepended.\n * @property {number} [presence=lunr.Query.presence.OPTIONAL] - The terms presence in any matching documents.\n */\n\n/**\n * Adds a {@link lunr.Query~Clause} to this query.\n *\n * Unless the clause contains the fields to be matched all fields will be matched. In addition\n * a default boost of 1 is applied to the clause.\n *\n * @param {lunr.Query~Clause} clause - The clause to add to this query.\n * @see lunr.Query~Clause\n * @returns {lunr.Query}\n */\nlunr.Query.prototype.clause = function (clause) {\n  if (!('fields' in clause)) {\n    clause.fields = this.allFields\n  }\n\n  if (!('boost' in clause)) {\n    clause.boost = 1\n  }\n\n  if (!('usePipeline' in clause)) {\n    clause.usePipeline = true\n  }\n\n  if (!('wildcard' in clause)) {\n    clause.wildcard = lunr.Query.wildcard.NONE\n  }\n\n  if ((clause.wildcard & lunr.Query.wildcard.LEADING) && (clause.term.charAt(0) != lunr.Query.wildcard)) {\n    clause.term = \"*\" + clause.term\n  }\n\n  if ((clause.wildcard & lunr.Query.wildcard.TRAILING) && (clause.term.slice(-1) != lunr.Query.wildcard)) {\n    clause.term = \"\" + clause.term + \"*\"\n  }\n\n  if (!('presence' in clause)) {\n    clause.presence = lunr.Query.presence.OPTIONAL\n  }\n\n  this.clauses.push(clause)\n\n  return this\n}\n\n/**\n * A negated query is one in which every clause has a presence of\n * prohibited. These queries require some special processing to return\n * the expected results.\n *\n * @returns boolean\n */\nlunr.Query.prototype.isNegated = function () {\n  for (var i = 0; i < this.clauses.length; i++) {\n    if (this.clauses[i].presence != lunr.Query.presence.PROHIBITED) {\n      return false\n    }\n  }\n\n  return true\n}\n\n/**\n * Adds a term to the current query, under the covers this will create a {@link lunr.Query~Clause}\n * to the list of clauses that make up this query.\n *\n * The term is used as is, i.e. no tokenization will be performed by this method. Instead conversion\n * to a token or token-like string should be done before calling this method.\n *\n * The term will be converted to a string by calling `toString`. Multiple terms can be passed as an\n * array, each term in the array will share the same options.\n *\n * @param {object|object[]} term - The term(s) to add to the query.\n * @param {object} [options] - Any additional properties to add to the query clause.\n * @returns {lunr.Query}\n * @see lunr.Query#clause\n * @see lunr.Query~Clause\n * @example <caption>adding a single term to a query</caption>\n * query.term(\"foo\")\n * @example <caption>adding a single term to a query and specifying search fields, term boost and automatic trailing wildcard</caption>\n * query.term(\"foo\", {\n *   fields: [\"title\"],\n *   boost: 10,\n *   wildcard: lunr.Query.wildcard.TRAILING\n * })\n * @example <caption>using lunr.tokenizer to convert a string to tokens before using them as terms</caption>\n * query.term(lunr.tokenizer(\"foo bar\"))\n */\nlunr.Query.prototype.term = function (term, options) {\n  if (Array.isArray(term)) {\n    term.forEach(function (t) { this.term(t, lunr.utils.clone(options)) }, this)\n    return this\n  }\n\n  var clause = options || {}\n  clause.term = term.toString()\n\n  this.clause(clause)\n\n  return this\n}\nlunr.QueryParseError = function (message, start, end) {\n  this.name = \"QueryParseError\"\n  this.message = message\n  this.start = start\n  this.end = end\n}\n\nlunr.QueryParseError.prototype = new Error\nlunr.QueryLexer = function (str) {\n  this.lexemes = []\n  this.str = str\n  this.length = str.length\n  this.pos = 0\n  this.start = 0\n  this.escapeCharPositions = []\n}\n\nlunr.QueryLexer.prototype.run = function () {\n  var state = lunr.QueryLexer.lexText\n\n  while (state) {\n    state = state(this)\n  }\n}\n\nlunr.QueryLexer.prototype.sliceString = function () {\n  var subSlices = [],\n      sliceStart = this.start,\n      sliceEnd = this.pos\n\n  for (var i = 0; i < this.escapeCharPositions.length; i++) {\n    sliceEnd = this.escapeCharPositions[i]\n    subSlices.push(this.str.slice(sliceStart, sliceEnd))\n    sliceStart = sliceEnd + 1\n  }\n\n  subSlices.push(this.str.slice(sliceStart, this.pos))\n  this.escapeCharPositions.length = 0\n\n  return subSlices.join('')\n}\n\nlunr.QueryLexer.prototype.emit = function (type) {\n  this.lexemes.push({\n    type: type,\n    str: this.sliceString(),\n    start: this.start,\n    end: this.pos\n  })\n\n  this.start = this.pos\n}\n\nlunr.QueryLexer.prototype.escapeCharacter = function () {\n  this.escapeCharPositions.push(this.pos - 1)\n  this.pos += 1\n}\n\nlunr.QueryLexer.prototype.next = function () {\n  if (this.pos >= this.length) {\n    return lunr.QueryLexer.EOS\n  }\n\n  var char = this.str.charAt(this.pos)\n  this.pos += 1\n  return char\n}\n\nlunr.QueryLexer.prototype.width = function () {\n  return this.pos - this.start\n}\n\nlunr.QueryLexer.prototype.ignore = function () {\n  if (this.start == this.pos) {\n    this.pos += 1\n  }\n\n  this.start = this.pos\n}\n\nlunr.QueryLexer.prototype.backup = function () {\n  this.pos -= 1\n}\n\nlunr.QueryLexer.prototype.acceptDigitRun = function () {\n  var char, charCode\n\n  do {\n    char = this.next()\n    charCode = char.charCodeAt(0)\n  } while (charCode > 47 && charCode < 58)\n\n  if (char != lunr.QueryLexer.EOS) {\n    this.backup()\n  }\n}\n\nlunr.QueryLexer.prototype.more = function () {\n  return this.pos < this.length\n}\n\nlunr.QueryLexer.EOS = 'EOS'\nlunr.QueryLexer.FIELD = 'FIELD'\nlunr.QueryLexer.TERM = 'TERM'\nlunr.QueryLexer.EDIT_DISTANCE = 'EDIT_DISTANCE'\nlunr.QueryLexer.BOOST = 'BOOST'\nlunr.QueryLexer.PRESENCE = 'PRESENCE'\n\nlunr.QueryLexer.lexField = function (lexer) {\n  lexer.backup()\n  lexer.emit(lunr.QueryLexer.FIELD)\n  lexer.ignore()\n  return lunr.QueryLexer.lexText\n}\n\nlunr.QueryLexer.lexTerm = function (lexer) {\n  if (lexer.width() > 1) {\n    lexer.backup()\n    lexer.emit(lunr.QueryLexer.TERM)\n  }\n\n  lexer.ignore()\n\n  if (lexer.more()) {\n    return lunr.QueryLexer.lexText\n  }\n}\n\nlunr.QueryLexer.lexEditDistance = function (lexer) {\n  lexer.ignore()\n  lexer.acceptDigitRun()\n  lexer.emit(lunr.QueryLexer.EDIT_DISTANCE)\n  return lunr.QueryLexer.lexText\n}\n\nlunr.QueryLexer.lexBoost = function (lexer) {\n  lexer.ignore()\n  lexer.acceptDigitRun()\n  lexer.emit(lunr.QueryLexer.BOOST)\n  return lunr.QueryLexer.lexText\n}\n\nlunr.QueryLexer.lexEOS = function (lexer) {\n  if (lexer.width() > 0) {\n    lexer.emit(lunr.QueryLexer.TERM)\n  }\n}\n\n// This matches the separator used when tokenising fields\n// within a document. These should match otherwise it is\n// not possible to search for some tokens within a document.\n//\n// It is possible for the user to change the separator on the\n// tokenizer so it _might_ clash with any other of the special\n// characters already used within the search string, e.g. :.\n//\n// This means that it is possible to change the separator in\n// such a way that makes some words unsearchable using a search\n// string.\nlunr.QueryLexer.termSeparator = lunr.tokenizer.separator\n\nlunr.QueryLexer.lexText = function (lexer) {\n  while (true) {\n    var char = lexer.next()\n\n    if (char == lunr.QueryLexer.EOS) {\n      return lunr.QueryLexer.lexEOS\n    }\n\n    // Escape character is '\\'\n    if (char.charCodeAt(0) == 92) {\n      lexer.escapeCharacter()\n      continue\n    }\n\n    if (char == \":\") {\n      return lunr.QueryLexer.lexField\n    }\n\n    if (char == \"~\") {\n      lexer.backup()\n      if (lexer.width() > 0) {\n        lexer.emit(lunr.QueryLexer.TERM)\n      }\n      return lunr.QueryLexer.lexEditDistance\n    }\n\n    if (char == \"^\") {\n      lexer.backup()\n      if (lexer.width() > 0) {\n        lexer.emit(lunr.QueryLexer.TERM)\n      }\n      return lunr.QueryLexer.lexBoost\n    }\n\n    // \"+\" indicates term presence is required\n    // checking for length to ensure that only\n    // leading \"+\" are considered\n    if (char == \"+\" && lexer.width() === 1) {\n      lexer.emit(lunr.QueryLexer.PRESENCE)\n      return lunr.QueryLexer.lexText\n    }\n\n    // \"-\" indicates term presence is prohibited\n    // checking for length to ensure that only\n    // leading \"-\" are considered\n    if (char == \"-\" && lexer.width() === 1) {\n      lexer.emit(lunr.QueryLexer.PRESENCE)\n      return lunr.QueryLexer.lexText\n    }\n\n    if (char.match(lunr.QueryLexer.termSeparator)) {\n      return lunr.QueryLexer.lexTerm\n    }\n  }\n}\n\nlunr.QueryParser = function (str, query) {\n  this.lexer = new lunr.QueryLexer (str)\n  this.query = query\n  this.currentClause = {}\n  this.lexemeIdx = 0\n}\n\nlunr.QueryParser.prototype.parse = function () {\n  this.lexer.run()\n  this.lexemes = this.lexer.lexemes\n\n  var state = lunr.QueryParser.parseClause\n\n  while (state) {\n    state = state(this)\n  }\n\n  return this.query\n}\n\nlunr.QueryParser.prototype.peekLexeme = function () {\n  return this.lexemes[this.lexemeIdx]\n}\n\nlunr.QueryParser.prototype.consumeLexeme = function () {\n  var lexeme = this.peekLexeme()\n  this.lexemeIdx += 1\n  return lexeme\n}\n\nlunr.QueryParser.prototype.nextClause = function () {\n  var completedClause = this.currentClause\n  this.query.clause(completedClause)\n  this.currentClause = {}\n}\n\nlunr.QueryParser.parseClause = function (parser) {\n  var lexeme = parser.peekLexeme()\n\n  if (lexeme == undefined) {\n    return\n  }\n\n  switch (lexeme.type) {\n    case lunr.QueryLexer.PRESENCE:\n      return lunr.QueryParser.parsePresence\n    case lunr.QueryLexer.FIELD:\n      return lunr.QueryParser.parseField\n    case lunr.QueryLexer.TERM:\n      return lunr.QueryParser.parseTerm\n    default:\n      var errorMessage = \"expected either a field or a term, found \" + lexeme.type\n\n      if (lexeme.str.length >= 1) {\n        errorMessage += \" with value '\" + lexeme.str + \"'\"\n      }\n\n      throw new lunr.QueryParseError (errorMessage, lexeme.start, lexeme.end)\n  }\n}\n\nlunr.QueryParser.parsePresence = function (parser) {\n  var lexeme = parser.consumeLexeme()\n\n  if (lexeme == undefined) {\n    return\n  }\n\n  switch (lexeme.str) {\n    case \"-\":\n      parser.currentClause.presence = lunr.Query.presence.PROHIBITED\n      break\n    case \"+\":\n      parser.currentClause.presence = lunr.Query.presence.REQUIRED\n      break\n    default:\n      var errorMessage = \"unrecognised presence operator'\" + lexeme.str + \"'\"\n      throw new lunr.QueryParseError (errorMessage, lexeme.start, lexeme.end)\n  }\n\n  var nextLexeme = parser.peekLexeme()\n\n  if (nextLexeme == undefined) {\n    var errorMessage = \"expecting term or field, found nothing\"\n    throw new lunr.QueryParseError (errorMessage, lexeme.start, lexeme.end)\n  }\n\n  switch (nextLexeme.type) {\n    case lunr.QueryLexer.FIELD:\n      return lunr.QueryParser.parseField\n    case lunr.QueryLexer.TERM:\n      return lunr.QueryParser.parseTerm\n    default:\n      var errorMessage = \"expecting term or field, found '\" + nextLexeme.type + \"'\"\n      throw new lunr.QueryParseError (errorMessage, nextLexeme.start, nextLexeme.end)\n  }\n}\n\nlunr.QueryParser.parseField = function (parser) {\n  var lexeme = parser.consumeLexeme()\n\n  if (lexeme == undefined) {\n    return\n  }\n\n  if (parser.query.allFields.indexOf(lexeme.str) == -1) {\n    var possibleFields = parser.query.allFields.map(function (f) { return \"'\" + f + \"'\" }).join(', '),\n        errorMessage = \"unrecognised field '\" + lexeme.str + \"', possible fields: \" + possibleFields\n\n    throw new lunr.QueryParseError (errorMessage, lexeme.start, lexeme.end)\n  }\n\n  parser.currentClause.fields = [lexeme.str]\n\n  var nextLexeme = parser.peekLexeme()\n\n  if (nextLexeme == undefined) {\n    var errorMessage = \"expecting term, found nothing\"\n    throw new lunr.QueryParseError (errorMessage, lexeme.start, lexeme.end)\n  }\n\n  switch (nextLexeme.type) {\n    case lunr.QueryLexer.TERM:\n      return lunr.QueryParser.parseTerm\n    default:\n      var errorMessage = \"expecting term, found '\" + nextLexeme.type + \"'\"\n      throw new lunr.QueryParseError (errorMessage, nextLexeme.start, nextLexeme.end)\n  }\n}\n\nlunr.QueryParser.parseTerm = function (parser) {\n  var lexeme = parser.consumeLexeme()\n\n  if (lexeme == undefined) {\n    return\n  }\n\n  parser.currentClause.term = lexeme.str.toLowerCase()\n\n  if (lexeme.str.indexOf(\"*\") != -1) {\n    parser.currentClause.usePipeline = false\n  }\n\n  var nextLexeme = parser.peekLexeme()\n\n  if (nextLexeme == undefined) {\n    parser.nextClause()\n    return\n  }\n\n  switch (nextLexeme.type) {\n    case lunr.QueryLexer.TERM:\n      parser.nextClause()\n      return lunr.QueryParser.parseTerm\n    case lunr.QueryLexer.FIELD:\n      parser.nextClause()\n      return lunr.QueryParser.parseField\n    case lunr.QueryLexer.EDIT_DISTANCE:\n      return lunr.QueryParser.parseEditDistance\n    case lunr.QueryLexer.BOOST:\n      return lunr.QueryParser.parseBoost\n    case lunr.QueryLexer.PRESENCE:\n      parser.nextClause()\n      return lunr.QueryParser.parsePresence\n    default:\n      var errorMessage = \"Unexpected lexeme type '\" + nextLexeme.type + \"'\"\n      throw new lunr.QueryParseError (errorMessage, nextLexeme.start, nextLexeme.end)\n  }\n}\n\nlunr.QueryParser.parseEditDistance = function (parser) {\n  var lexeme = parser.consumeLexeme()\n\n  if (lexeme == undefined) {\n    return\n  }\n\n  var editDistance = parseInt(lexeme.str, 10)\n\n  if (isNaN(editDistance)) {\n    var errorMessage = \"edit distance must be numeric\"\n    throw new lunr.QueryParseError (errorMessage, lexeme.start, lexeme.end)\n  }\n\n  parser.currentClause.editDistance = editDistance\n\n  var nextLexeme = parser.peekLexeme()\n\n  if (nextLexeme == undefined) {\n    parser.nextClause()\n    return\n  }\n\n  switch (nextLexeme.type) {\n    case lunr.QueryLexer.TERM:\n      parser.nextClause()\n      return lunr.QueryParser.parseTerm\n    case lunr.QueryLexer.FIELD:\n      parser.nextClause()\n      return lunr.QueryParser.parseField\n    case lunr.QueryLexer.EDIT_DISTANCE:\n      return lunr.QueryParser.parseEditDistance\n    case lunr.QueryLexer.BOOST:\n      return lunr.QueryParser.parseBoost\n    case lunr.QueryLexer.PRESENCE:\n      parser.nextClause()\n      return lunr.QueryParser.parsePresence\n    default:\n      var errorMessage = \"Unexpected lexeme type '\" + nextLexeme.type + \"'\"\n      throw new lunr.QueryParseError (errorMessage, nextLexeme.start, nextLexeme.end)\n  }\n}\n\nlunr.QueryParser.parseBoost = function (parser) {\n  var lexeme = parser.consumeLexeme()\n\n  if (lexeme == undefined) {\n    return\n  }\n\n  var boost = parseInt(lexeme.str, 10)\n\n  if (isNaN(boost)) {\n    var errorMessage = \"boost must be numeric\"\n    throw new lunr.QueryParseError (errorMessage, lexeme.start, lexeme.end)\n  }\n\n  parser.currentClause.boost = boost\n\n  var nextLexeme = parser.peekLexeme()\n\n  if (nextLexeme == undefined) {\n    parser.nextClause()\n    return\n  }\n\n  switch (nextLexeme.type) {\n    case lunr.QueryLexer.TERM:\n      parser.nextClause()\n      return lunr.QueryParser.parseTerm\n    case lunr.QueryLexer.FIELD:\n      parser.nextClause()\n      return lunr.QueryParser.parseField\n    case lunr.QueryLexer.EDIT_DISTANCE:\n      return lunr.QueryParser.parseEditDistance\n    case lunr.QueryLexer.BOOST:\n      return lunr.QueryParser.parseBoost\n    case lunr.QueryLexer.PRESENCE:\n      parser.nextClause()\n      return lunr.QueryParser.parsePresence\n    default:\n      var errorMessage = \"Unexpected lexeme type '\" + nextLexeme.type + \"'\"\n      throw new lunr.QueryParseError (errorMessage, nextLexeme.start, nextLexeme.end)\n  }\n}\n\n  /**\n   * export the module via AMD, CommonJS or as a browser global\n   * Export code from https://github.com/umdjs/umd/blob/master/returnExports.js\n   */\n  ;(function (root, factory) {\n    if (typeof define === 'function' && define.amd) {\n      // AMD. Register as an anonymous module.\n      define(factory)\n    } else if (typeof exports === 'object') {\n      /**\n       * Node. Does not work with strict CommonJS, but\n       * only CommonJS-like enviroments that support module.exports,\n       * like Node.\n       */\n      module.exports = factory()\n    } else {\n      // Browser globals (root is window)\n      root.lunr = factory()\n    }\n  }(this, function () {\n    /**\n     * Just return a value to define the module export.\n     * This example returns an object, but the module\n     * can return a function as the exported value.\n     */\n    return lunr\n  }))\n})();\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A RTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport lunr from \"lunr\"\n\nimport { getElement } from \"~/browser/element/_\"\nimport \"~/polyfills\"\n\nimport { Search } from \"../../_\"\nimport { SearchConfig } from \"../../config\"\nimport {\n  SearchMessage,\n  SearchMessageType\n} from \"../message\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Add support for `iframe-worker` shim\n *\n * While `importScripts` is synchronous when executed inside of a web worker,\n * it's not possible to provide a synchronous shim implementation. The cool\n * thing is that awaiting a non-Promise will convert it into a Promise, so\n * extending the type definition to return a `Promise` shouldn't break anything.\n *\n * @see https://bit.ly/2PjDnXi - GitHub comment\n *\n * @param urls - Scripts to load\n *\n * @returns Promise resolving with no result\n */\ndeclare global {\n  function importScripts(...urls: string[]): Promise<void> | void\n}\n\n/* ----------------------------------------------------------------------------\n * Data\n * ------------------------------------------------------------------------- */\n\n/**\n * Search index\n */\nlet index: Search\n\n/* ----------------------------------------------------------------------------\n * Helper functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Fetch (= import) multi-language support through `lunr-languages`\n *\n * This function automatically imports the stemmers necessary to process the\n * languages which are defined as part of the search configuration.\n *\n * If the worker runs inside of an `iframe` (when using `iframe-worker` as\n * a shim), the base URL for the stemmers to be loaded must be determined by\n * searching for the first `script` element with a `src` attribute, which will\n * contain the contents of this script.\n *\n * @param config - Search configuration\n *\n * @returns Promise resolving with no result\n */\nasync function setupSearchLanguages(\n  config: SearchConfig\n): Promise<void> {\n  let base = \"../lunr\"\n\n  /* Detect `iframe-worker` and fix base URL */\n  if (typeof parent !== \"undefined\" && \"IFrameWorker\" in parent) {\n    const worker = getElement<HTMLScriptElement>(\"script[src]\")!\n    const [path] = worker.src.split(\"/worker\")\n\n    /* Prefix base with path */\n    base = base.replace(\"..\", path)\n  }\n\n  /* Add scripts for languages */\n  const scripts = []\n  for (const lang of config.lang) {\n    switch (lang) {\n\n      /* Add segmenter for Japanese */\n      case \"ja\":\n        scripts.push(`${base}/tinyseg.js`)\n        break\n\n      /* Add segmenter for Hindi and Thai */\n      case \"hi\":\n      case \"th\":\n        scripts.push(`${base}/wordcut.js`)\n        break\n    }\n\n    /* Add language support */\n    if (lang !== \"en\")\n      scripts.push(`${base}/min/lunr.${lang}.min.js`)\n  }\n\n  /* Add multi-language support */\n  if (config.lang.length > 1)\n    scripts.push(`${base}/min/lunr.multi.min.js`)\n\n  /* Load scripts synchronously */\n  if (scripts.length)\n    await importScripts(\n      `${base}/min/lunr.stemmer.support.min.js`,\n      ...scripts\n    )\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Message handler\n *\n * @param message - Source message\n *\n * @returns Target message\n */\nexport async function handler(\n  message: SearchMessage\n): Promise<SearchMessage> {\n  switch (message.type) {\n\n    /* Search setup message */\n    case SearchMessageType.SETUP:\n      await setupSearchLanguages(message.data.config)\n      index = new Search(message.data)\n      return {\n        type: SearchMessageType.READY\n      }\n\n    /* Search query message */\n    case SearchMessageType.QUERY:\n      const query = message.data\n      try {\n        return {\n          type: SearchMessageType.RESULT,\n          data: index.search(query)\n        }\n\n      /* Return empty result in case of error */\n      } catch (err) {\n        console.warn(`Invalid query: ${query} \u2013 see https://bit.ly/2s3ChXG`)\n        console.warn(err)\n        return {\n          type: SearchMessageType.RESULT,\n          data: { items: [] }\n        }\n      }\n\n    /* All other messages */\n    default:\n      throw new TypeError(\"Invalid message type\")\n  }\n}\n\n/* ----------------------------------------------------------------------------\n * Worker\n * ------------------------------------------------------------------------- */\n\n/* Expose Lunr.js in global scope, or stemmers won't work */\nself.lunr = lunr\n\n/* Handle messages */\naddEventListener(\"message\", async ev => {\n  postMessage(await handler(ev.data))\n})\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Retrieve all elements matching the query selector\n *\n * @template T - Element type\n *\n * @param selector - Query selector\n * @param node - Node of reference\n *\n * @returns Elements\n */\nexport function getElements<T extends keyof HTMLElementTagNameMap>(\n  selector: T, node?: ParentNode\n): HTMLElementTagNameMap[T][]\n\nexport function getElements<T extends HTMLElement>(\n  selector: string, node?: ParentNode\n): T[]\n\nexport function getElements<T extends HTMLElement>(\n  selector: string, node: ParentNode = document\n): T[] {\n  return Array.from(node.querySelectorAll<T>(selector))\n}\n\n/**\n * Retrieve an element matching a query selector or throw a reference error\n *\n * Note that this function assumes that the element is present. If unsure if an\n * element is existent, use the `getOptionalElement` function instead.\n *\n * @template T - Element type\n *\n * @param selector - Query selector\n * @param node - Node of reference\n *\n * @returns Element\n */\nexport function getElement<T extends keyof HTMLElementTagNameMap>(\n  selector: T, node?: ParentNode\n): HTMLElementTagNameMap[T]\n\nexport function getElement<T extends HTMLElement>(\n  selector: string, node?: ParentNode\n): T\n\nexport function getElement<T extends HTMLElement>(\n  selector: string, node: ParentNode = document\n): T {\n  const el = getOptionalElement<T>(selector, node)\n  if (typeof el === \"undefined\")\n    throw new ReferenceError(\n      `Missing element: expected \"${selector}\" to be present`\n    )\n\n  /* Return element */\n  return el\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Retrieve an optional element matching the query selector\n *\n * @template T - Element type\n *\n * @param selector - Query selector\n * @param node - Node of reference\n *\n * @returns Element or nothing\n */\nexport function getOptionalElement<T extends keyof HTMLElementTagNameMap>(\n  selector: T, node?: ParentNode\n): HTMLElementTagNameMap[T] | undefined\n\nexport function getOptionalElement<T extends HTMLElement>(\n  selector: string, node?: ParentNode\n): T | undefined\n\nexport function getOptionalElement<T extends HTMLElement>(\n  selector: string, node: ParentNode = document\n): T | undefined {\n  return node.querySelector<T>(selector) || undefined\n}\n\n/**\n * Retrieve the currently active element\n *\n * @returns Element or nothing\n */\nexport function getActiveElement(): HTMLElement | undefined {\n  return (\n    document.activeElement?.shadowRoot?.activeElement as HTMLElement ??\n    document.activeElement as HTMLElement ??\n    undefined\n  )\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Polyfills\n * ------------------------------------------------------------------------- */\n\n/* Polyfill `Object.entries` */\nif (!Object.entries)\n  Object.entries = function (obj: object) {\n    const data: [string, string][] = []\n    for (const key of Object.keys(obj))\n      // @ts-expect-error - ignore property access warning\n      data.push([key, obj[key]])\n\n    /* Return entries */\n    return data\n  }\n\n/* Polyfill `Object.values` */\nif (!Object.values)\n  Object.values = function (obj: object) {\n    const data: string[] = []\n    for (const key of Object.keys(obj))\n      // @ts-expect-error - ignore property access warning\n      data.push(obj[key])\n\n    /* Return values */\n    return data\n  }\n\n/* ------------------------------------------------------------------------- */\n\n/* Polyfills for `Element` */\nif (typeof Element !== \"undefined\") {\n\n  /* Polyfill `Element.scrollTo` */\n  if (!Element.prototype.scrollTo)\n    Element.prototype.scrollTo = function (\n      x?: ScrollToOptions | number, y?: number\n    ): void {\n      if (typeof x === \"object\") {\n        this.scrollLeft = x.left!\n        this.scrollTop = x.top!\n      } else {\n        this.scrollLeft = x!\n        this.scrollTop = y!\n      }\n    }\n\n  /* Polyfill `Element.replaceWith` */\n  if (!Element.prototype.replaceWith)\n    Element.prototype.replaceWith = function (\n      ...nodes: Array<string | Node>\n    ): void {\n      const parent = this.parentNode\n      if (parent) {\n        if (nodes.length === 0)\n          parent.removeChild(this)\n\n        /* Replace children and create text nodes */\n        for (let i = nodes.length - 1; i >= 0; i--) {\n          let node = nodes[i]\n          if (typeof node === \"string\")\n            node = document.createTextNode(node)\n          else if (node.parentNode)\n            node.parentNode.removeChild(node)\n\n          /* Replace child or insert before previous sibling */\n          if (!i)\n            parent.replaceChild(node, this)\n          else\n            parent.insertBefore(this.previousSibling!, node)\n        }\n      }\n    }\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search configuration\n */\nexport interface SearchConfig {\n  lang: string[]                       /* Search languages */\n  separator: string                    /* Search separator */\n  pipeline: SearchPipelineFn[]         /* Search pipeline */\n}\n\n/**\n * Search document\n */\nexport interface SearchDocument {\n  location: string                     /* Document location */\n  title: string                        /* Document title */\n  text: string                         /* Document text */\n  tags?: string[]                      /* Document tags */\n  boost?: number                       /* Document boost */\n  parent?: SearchDocument              /* Document parent */\n}\n\n/**\n * Search options\n */\nexport interface SearchOptions {\n  suggest: boolean                     /* Search suggestions */\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Search index\n */\nexport interface SearchIndex {\n  config: SearchConfig                 /* Search configuration */\n  docs: SearchDocument[]               /* Search documents */\n  options: SearchOptions               /* Search options */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search pipeline function\n */\ntype SearchPipelineFn =\n  | \"trimmer\"                          /* Trimmer */\n  | \"stopWordFilter\"                   /* Stop word filter */\n  | \"stemmer\"                          /* Stemmer */\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Create a search document map\n *\n * This function creates a mapping of URLs (including anchors) to the actual\n * articles and sections. It relies on the invariant that the search index is\n * ordered with the main article appearing before all sections with anchors.\n * If this is not the case, the logic music be changed.\n *\n * @param docs - Search documents\n *\n * @returns Search document map\n */\nexport function setupSearchDocumentMap(\n  docs: SearchDocument[]\n): Map<string, SearchDocument> {\n  const map = new Map<string, SearchDocument>()\n  for (const doc of docs) {\n    const [path] = doc.location.split(\"#\")\n\n    /* Add document article */\n    const article = map.get(path)\n    if (typeof article === \"undefined\") {\n      map.set(path, doc)\n\n      /* Add document section */\n    } else {\n      map.set(doc.location, doc)\n      doc.parent = article\n    }\n  }\n\n  /* Return search document map */\n  return map\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Visitor function\n *\n * @param start - Start offset\n * @param end - End offset\n */\ntype VisitorFn = (\n  start: number, end: number\n) => void\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Split a string using the given separator\n *\n * @param input - Input value\n * @param separator - Separator\n * @param fn - Visitor function\n */\nexport function split(\n  input: string, separator: RegExp, fn: VisitorFn\n): void {\n  separator = new RegExp(separator, \"g\")\n\n  /* Split string using separator */\n  let match: RegExpExecArray | null\n  let index = 0\n  do {\n    match = separator.exec(input)\n\n    /* Emit non-empty range */\n    const until = match?.index ?? input.length\n    if (index < until)\n      fn(index, until)\n\n    /* Update last index */\n    if (match) {\n      const [term] = match\n      index = match.index + term.length\n\n      /* Support zero-length lookaheads */\n      if (term.length === 0)\n        separator.lastIndex = match.index + 1\n    }\n  } while (match)\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Extraction type\n *\n * This type defines the possible values that are encoded into the first two\n * bits of a section that is part of the blocks of a tokenization table. There\n * are three types of interest: HTML opening and closing tags, as well as the\n * actual text content we need to extract for indexing.\n */\nexport const enum Extract {\n  TAG_OPEN  = 0,                       /* HTML opening tag */\n  TEXT      = 1,                       /* Text content */\n  TAG_CLOSE = 2                        /* HTML closing tag */\n}\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Visitor function\n *\n * @param block - Block index\n * @param type - Extraction type\n * @param start - Start offset\n * @param end - End offset\n */\ntype VisitorFn = (\n  block: number, type: Extract, start: number, end: number\n) => void\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Split a string into markup and text sections\n *\n * This function scans a string and divides it up into sections of markup and\n * text. For each section, it invokes the given visitor function with the block\n * index, extraction type, as well as start and end offsets. Using a visitor\n * function (= streaming data) is ideal for minimizing pressure on the GC.\n *\n * @param input - Input value\n * @param fn - Visitor function\n */\nexport function extract(\n  input: string, fn: VisitorFn\n): void {\n\n  let block = 0                        /* Current block */\n  let start = 0                        /* Current start offset */\n  let end = 0                          /* Current end offset */\n\n  /* Split string into sections */\n  for (let stack = 0; end < input.length; end++) {\n\n    /* Opening tag after non-empty section */\n    if (input.charAt(end) === \"<\" && end > start) {\n      fn(block, Extract.TEXT, start, start = end)\n\n    /* Closing tag */\n    } else if (input.charAt(end) === \">\") {\n      if (input.charAt(start + 1) === \"/\") {\n        if (--stack === 0)\n          fn(block++, Extract.TAG_CLOSE, start, end + 1)\n\n      /* Tag is not self-closing */\n      } else if (input.charAt(end - 1) !== \"/\") {\n        if (stack++ === 0)\n          fn(block, Extract.TAG_OPEN, start, end + 1)\n      }\n\n      /* New section */\n      start = end + 1\n    }\n  }\n\n  /* Add trailing section */\n  if (end > start)\n    fn(block, Extract.TEXT, start, end)\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Position table\n */\nexport type PositionTable = number[][]\n\n/**\n * Position\n */\nexport type Position = number\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Highlight all occurrences in a string\n *\n * This function receives a field's value (e.g. like `title` or `text`), it's\n * position table that was generated during indexing, and the positions found\n * when executing the query. It then highlights all occurrences, and returns\n * their concatenation. In case of multiple blocks, two are returned.\n *\n * @param input - Input value\n * @param table - Table for indexing\n * @param positions - Occurrences\n * @param full - Full results\n *\n * @returns Highlighted string value\n */\nexport function highlight(\n  input: string, table: PositionTable, positions: Position[], full = false\n): string {\n  return highlightAll([input], table, positions, full).pop()!\n}\n\n/**\n * Highlight all occurrences in a set of strings\n *\n * @param inputs - Input values\n * @param table - Table for indexing\n * @param positions - Occurrences\n * @param full - Full results\n *\n * @returns Highlighted string values\n */\nexport function highlightAll(\n  inputs: string[], table: PositionTable, positions: Position[], full = false\n): string[] {\n\n  /* Map blocks to input values */\n  const mapping = [0]\n  for (let t = 1; t < table.length; t++) {\n    const prev = table[t - 1]\n    const next = table[t]\n\n    /* Check if table points to new block */\n    const p = prev[prev.length - 1] >>> 2 & 0x3FF\n    const q = next[0]               >>> 12\n\n    /* Add block to mapping */\n    mapping.push(+(p > q) + mapping[mapping.length - 1])\n  }\n\n  /* Highlight strings one after another */\n  return inputs.map((input, i) => {\n    let cursor = 0\n\n    /* Map occurrences to blocks */\n    const blocks = new Map<number, number[]>()\n    for (const p of positions.sort((a, b) => a - b)) {\n      const index = p & 0xFFFFF\n      const block = p >>> 20\n      if (mapping[block] !== i)\n        continue\n\n      /* Ensure presence of block group */\n      let group = blocks.get(block)\n      if (typeof group === \"undefined\")\n        blocks.set(block, group = [])\n\n      /* Add index to group */\n      group.push(index)\n    }\n\n    /* Just return string, if no occurrences */\n    if (blocks.size === 0)\n      return input\n\n    /* Compute slices */\n    const slices: string[] = []\n    for (const [block, indexes] of blocks) {\n      const t = table[block]\n\n      /* Extract positions and length */\n      const start  = t[0]            >>> 12\n      const end    = t[t.length - 1] >>> 12\n      const length = t[t.length - 1] >>> 2 & 0x3FF\n\n      /* Add prefix, if full results are desired */\n      if (full && start > cursor)\n        slices.push(input.slice(cursor, start))\n\n      /* Extract and highlight slice */\n      let slice = input.slice(start, end + length)\n      for (const j of indexes.sort((a, b) => b - a)) {\n\n        /* Retrieve offset and length of match */\n        const p = (t[j] >>> 12) - start\n        const q = (t[j] >>> 2 & 0x3FF) + p\n\n        /* Wrap occurrence */\n        slice = [\n          slice.slice(0, p),\n          \"<mark>\",\n          slice.slice(p, q),\n          \"</mark>\",\n          slice.slice(q)\n        ].join(\"\")\n      }\n\n      /* Update cursor */\n      cursor = end + length\n\n      /* Append slice and abort if we have two */\n      if (slices.push(slice) === 2)\n        break\n    }\n\n    /* Add suffix, if full results are desired */\n    if (full && cursor < input.length)\n      slices.push(input.slice(cursor))\n\n    /* Return highlighted slices */\n    return slices.join(\"\")\n  })\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { split } from \"../_\"\nimport {\n  Extract,\n  extract\n} from \"../extract\"\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Split a string or set of strings into tokens\n *\n * This tokenizer supersedes the default tokenizer that is provided by Lunr.js,\n * as it is aware of HTML tags and allows for multi-character splitting.\n *\n * It takes the given inputs, splits each of them into markup and text sections,\n * tokenizes and segments (if necessary) each of them, and then indexes them in\n * a table by using a compact bit representation. Bitwise techniques are used\n * to write and read from the table during indexing and querying.\n *\n * @see https://bit.ly/3W3Xw4J - Search: better, faster, smaller\n *\n * @param input - Input value(s)\n *\n * @returns Tokens\n */\nexport function tokenize(\n  input?: string | string[]\n): lunr.Token[] {\n  const tokens: lunr.Token[] = []\n  if (typeof input === \"undefined\")\n    return tokens\n\n  /* Tokenize strings one after another */\n  const inputs = Array.isArray(input) ? input : [input]\n  for (let i = 0; i < inputs.length; i++) {\n    const table = lunr.tokenizer.table\n    const total = table.length\n\n    /* Split string into sections and tokenize content blocks */\n    extract(inputs[i], (block, type, start, end) => {\n      table[block += total] ||= []\n      switch (type) {\n\n        /* Handle markup */\n        case Extract.TAG_OPEN:\n        case Extract.TAG_CLOSE:\n          table[block].push(\n            start       << 12 |\n            end - start <<  2 |\n            type\n          )\n          break\n\n        /* Handle text content */\n        case Extract.TEXT:\n          const section = inputs[i].slice(start, end)\n          split(section, lunr.tokenizer.separator, (index, until) => {\n\n            /**\n             * Apply segmenter after tokenization. Note that the segmenter will\n             * also split words at word boundaries, which is not what we want,\n             * so we need to check if we can somehow mitigate this behavior.\n             */\n            if (typeof lunr.segmenter !== \"undefined\") {\n              const subsection = section.slice(index, until)\n              if (/^[MHIK]$/.test(lunr.segmenter.ctype_(subsection))) {\n                const segments = lunr.segmenter.segment(subsection)\n                for (let s = 0, l = 0; s < segments.length; s++) {\n\n                  /* Add block to section */\n                  table[block] ||= []\n                  table[block].push(\n                    start + index + l  << 12 |\n                    segments[s].length <<  2 |\n                    type\n                  )\n\n                  /* Add token with position */\n                  tokens.push(new lunr.Token(\n                    segments[s].toLowerCase(), {\n                      position: block << 20 | table[block].length - 1\n                    }\n                  ))\n\n                  /* Keep track of length */\n                  l += segments[s].length\n                }\n                return\n              }\n            }\n\n            /* Add block to section */\n            table[block].push(\n              start + index << 12 |\n              until - index <<  2 |\n              type\n            )\n\n            /* Add token with position */\n            tokens.push(new lunr.Token(\n              section.slice(index, until).toLowerCase(), {\n                position: block << 20 | table[block].length - 1\n              }\n            ))\n          })\n      }\n    })\n  }\n\n  /* Return tokens */\n  return tokens\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Helper types\n * ------------------------------------------------------------------------- */\n\n/**\n * Visitor function\n *\n * @param value - String value\n *\n * @returns String term(s)\n */\ntype VisitorFn = (\n  value: string\n) => string | string[]\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Default transformation function\n *\n * 1. Trim excess whitespace from left and right.\n *\n * 2. Search for parts in quotation marks and prepend a `+` modifier to denote\n *    that the resulting document must contain all parts, converting the query\n *    to an `AND` query (as opposed to the default `OR` behavior). While users\n *    may expect parts enclosed in quotation marks to map to span queries, i.e.\n *    for which order is important, Lunr.js doesn't support them, so the best\n *    we can do is to convert the parts to an `AND` query.\n *\n * 3. Replace control characters which are not located at the beginning of the\n *    query or preceded by white space, or are not followed by a non-whitespace\n *    character or are at the end of the query string. Furthermore, filter\n *    unmatched quotation marks.\n *\n * 4. Split the query string at whitespace, then pass each part to the visitor\n *    function for tokenization, and append a wildcard to every resulting term\n *    that is not explicitly marked with a `+`, `-`, `~` or `^` modifier, since\n *    it ensures consistent and stable ranking when multiple terms are entered.\n *    Also, if a fuzzy or boost modifier are given, but no numeric value has\n *    been entered, default to 1 to not induce a query error.\n *\n * @param query - Query value\n * @param fn - Visitor function\n *\n * @returns Transformed query value\n */\nexport function transform(\n  query: string, fn: VisitorFn = term => term\n): string {\n  return query\n\n    /* => 1 */\n    .trim()\n\n    /* => 2 */\n    .split(/\"([^\"]+)\"/g)\n      .map((parts, index) => index & 1\n        ? parts.replace(/^\\b|^(?![^\\x00-\\x7F]|$)|\\s+/g, \" +\")\n        : parts\n      )\n      .join(\"\")\n\n    /* => 3 */\n    .replace(/\"|(?:^|\\s+)[*+\\-:^~]+(?=\\s+|$)/g, \"\")\n\n    /* => 4 */\n    .split(/\\s+/g)\n      .reduce((prev, term) => {\n        const next = fn(term)\n        return [...prev, ...Array.isArray(next) ? next : [next]]\n      }, [] as string[])\n      .map(term => /([~^]$)/.test(term) ? `${term}1` : term)\n      .map(term => /(^[+-]|[~^]\\d+$)/.test(term) ? term : `${term}*`)\n      .join(\" \")\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport { split } from \"../../internal\"\nimport { transform } from \"../transform\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search query clause\n */\nexport interface SearchQueryClause {\n  presence: lunr.Query.presence        /* Clause presence */\n  term: string                         /* Clause term */\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Search query terms\n */\nexport type SearchQueryTerms = Record<string, boolean>\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Transform search query\n *\n * This function lexes the given search query and applies the transformation\n * function to each term, preserving markup like `+` and `-` modifiers.\n *\n * @param query - Search query\n *\n * @returns Search query\n */\nexport function transformSearchQuery(\n  query: string\n): string {\n\n  /* Split query terms with tokenizer */\n  return transform(query, part => {\n    const terms: string[] = []\n\n    /* Initialize lexer and analyze part */\n    const lexer = new lunr.QueryLexer(part)\n    lexer.run()\n\n    /* Extract and tokenize term from lexeme */\n    for (const { type, str: term, start, end } of lexer.lexemes)\n      switch (type) {\n\n        /* Hack: remove colon - see https://bit.ly/3wD3T3I */\n        case \"FIELD\":\n          if (![\"title\", \"text\", \"tags\"].includes(term))\n            part = [\n              part.slice(0, end),\n              \" \",\n              part.slice(end + 1)\n            ].join(\"\")\n          break\n\n        /* Tokenize term */\n        case \"TERM\":\n          split(term, lunr.tokenizer.separator, (...range) => {\n            terms.push([\n              part.slice(0, start),\n              term.slice(...range),\n              part.slice(end)\n            ].join(\"\"))\n          })\n      }\n\n    /* Return terms */\n    return terms\n  })\n}\n\n/* ------------------------------------------------------------------------- */\n\n/**\n * Parse a search query for analysis\n *\n * Lunr.js itself has a bug where it doesn't detect or remove wildcards for\n * query clauses, so we must do this here.\n *\n * @see https://bit.ly/3DpTGtz - GitHub issue\n *\n * @param value - Query value\n *\n * @returns Search query clauses\n */\nexport function parseSearchQuery(\n  value: string\n): SearchQueryClause[] {\n  const query  = new lunr.Query([\"title\", \"text\", \"tags\"])\n  const parser = new lunr.QueryParser(value, query)\n\n  /* Parse Search query */\n  parser.parse()\n  for (const clause of query.clauses) {\n    clause.usePipeline = true\n\n    /* Handle leading wildcard */\n    if (clause.term.startsWith(\"*\")) {\n      clause.wildcard = lunr.Query.wildcard.LEADING\n      clause.term = clause.term.slice(1)\n    }\n\n    /* Handle trailing wildcard */\n    if (clause.term.endsWith(\"*\")) {\n      clause.wildcard = lunr.Query.wildcard.TRAILING\n      clause.term = clause.term.slice(0, -1)\n    }\n  }\n\n  /* Return query clauses */\n  return query.clauses\n}\n\n/**\n * Analyze the search query clauses in regard to the search terms found\n *\n * @param query - Search query clauses\n * @param terms - Search terms\n *\n * @returns Search query terms\n */\nexport function getSearchQueryTerms(\n  query: SearchQueryClause[], terms: string[]\n): SearchQueryTerms {\n  const clauses = new Set<SearchQueryClause>(query)\n\n  /* Match query clauses against terms */\n  const result: SearchQueryTerms = {}\n  for (let t = 0; t < terms.length; t++)\n    for (const clause of clauses)\n      if (terms[t].startsWith(clause.term)) {\n        result[clause.term] = true\n        clauses.delete(clause)\n      }\n\n  /* Annotate unmatched non-stopword query clauses */\n  for (const clause of clauses)\n    if (lunr.stopWordFilter?.(clause.term))\n      result[clause.term] = false\n\n  /* Return query terms */\n  return result\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Segment a search query using the inverted index\n *\n * This function implements a clever approach to text segmentation for Asian\n * languages, as it used the information already available in the search index.\n * The idea is to greedily segment the search query based on the tokens that are\n * already part of the index, as described in the linked issue.\n *\n * @see https://bit.ly/3lwjrk7 - GitHub issue\n *\n * @param query - Query value\n * @param index - Inverted index\n *\n * @returns Segmented query value\n */\nexport function segment(\n  query: string, index: object\n): Iterable<string> {\n  const segments = new Set<string>()\n\n  /* Segment search query */\n  const wordcuts = new Uint16Array(query.length)\n  for (let i = 0; i < query.length; i++)\n    for (let j = i + 1; j < query.length; j++) {\n      const value = query.slice(i, j)\n      if (value in index)\n        wordcuts[i] = j - i\n    }\n\n  /* Compute longest matches with minimum overlap */\n  const stack = [0]\n  for (let s = stack.length; s > 0;) {\n    const p = stack[--s]\n    for (let q = 1; q < wordcuts[p]; q++)\n      if (wordcuts[p + q] > wordcuts[p] - q) {\n        segments.add(query.slice(p, p + q))\n        stack[s++] = p + q\n      }\n\n    /* Continue at end of query string */\n    const q = p + wordcuts[p]\n    if (wordcuts[q] && q < query.length - 1)\n      stack[s++] = q\n\n    /* Add current segment */\n    segments.add(query.slice(p, q))\n  }\n\n  // @todo fix this case in the code block above, this is a hotfix\n  if (segments.has(\"\"))\n    return new Set([query])\n\n  /* Return segmented query value */\n  return segments\n}\n", "/*\n * Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport {\n  SearchDocument,\n  SearchIndex,\n  SearchOptions,\n  setupSearchDocumentMap\n} from \"../config\"\nimport {\n  Position,\n  PositionTable,\n  highlight,\n  highlightAll,\n  tokenize\n} from \"../internal\"\nimport {\n  SearchQueryTerms,\n  getSearchQueryTerms,\n  parseSearchQuery,\n  segment,\n  transformSearchQuery\n} from \"../query\"\n\n/* ----------------------------------------------------------------------------\n * Types\n * ------------------------------------------------------------------------- */\n\n/**\n * Search item\n */\nexport interface SearchItem\n  extends SearchDocument\n{\n  score: number                        /* Score (relevance) */\n  terms: SearchQueryTerms              /* Search query terms */\n}\n\n/**\n * Search result\n */\nexport interface SearchResult {\n  items: SearchItem[][]                /* Search items */\n  suggest?: string[]                   /* Search suggestions */\n}\n\n/* ----------------------------------------------------------------------------\n * Functions\n * ------------------------------------------------------------------------- */\n\n/**\n * Create field extractor factory\n *\n * @param table - Position table map\n *\n * @returns Extractor factory\n */\nfunction extractor(table: Map<string, PositionTable>) {\n  return (name: keyof SearchDocument) => {\n    return (doc: SearchDocument) => {\n      if (typeof doc[name] === \"undefined\")\n        return undefined\n\n      /* Compute identifier and initialize table */\n      const id = [doc.location, name].join(\":\")\n      table.set(id, lunr.tokenizer.table = [])\n\n      /* Return field value */\n      return doc[name]\n    }\n  }\n}\n\n/**\n * Compute the difference of two lists of strings\n *\n * @param a - 1st list of strings\n * @param b - 2nd list of strings\n *\n * @returns Difference\n */\nfunction difference(a: string[], b: string[]): string[] {\n  const [x, y] = [new Set(a), new Set(b)]\n  return [\n    ...new Set([...x].filter(value => !y.has(value)))\n  ]\n}\n\n/* ----------------------------------------------------------------------------\n * Class\n * ------------------------------------------------------------------------- */\n\n/**\n * Search index\n */\nexport class Search {\n\n  /**\n   * Search document map\n   */\n  protected map: Map<string, SearchDocument>\n\n  /**\n   * Search options\n   */\n  protected options: SearchOptions\n\n  /**\n   * The underlying Lunr.js search index\n   */\n  protected index: lunr.Index\n\n  /**\n   * Internal position table map\n   */\n  protected table: Map<string, PositionTable>\n\n  /**\n   * Create the search integration\n   *\n   * @param data - Search index\n   */\n  public constructor({ config, docs, options }: SearchIndex) {\n    const field = extractor(this.table = new Map())\n\n    /* Set up document map and options */\n    this.map = setupSearchDocumentMap(docs)\n    this.options = options\n\n    /* Set up document index */\n    this.index = lunr(function () {\n      this.metadataWhitelist = [\"position\"]\n      this.b(0)\n\n      /* Set up (multi-)language support */\n      if (config.lang.length === 1 && config.lang[0] !== \"en\") {\n        // @ts-expect-error - namespace indexing not supported\n        this.use(lunr[config.lang[0]])\n      } else if (config.lang.length > 1) {\n        this.use(lunr.multiLanguage(...config.lang))\n      }\n\n      /* Set up custom tokenizer (must be after language setup) */\n      this.tokenizer = tokenize as typeof lunr.tokenizer\n      lunr.tokenizer.separator = new RegExp(config.separator)\n\n      /* Set up custom segmenter, if loaded */\n      lunr.segmenter = \"TinySegmenter\" in lunr\n        ? new lunr.TinySegmenter()\n        : undefined\n\n      /* Compute functions to be removed from the pipeline */\n      const fns = difference([\n        \"trimmer\", \"stopWordFilter\", \"stemmer\"\n      ], config.pipeline)\n\n      /* Remove functions from the pipeline for registered languages */\n      for (const lang of config.lang.map(language => (\n        // @ts-expect-error - namespace indexing not supported\n        language === \"en\" ? lunr : lunr[language]\n      )))\n        for (const fn of fns) {\n          this.pipeline.remove(lang[fn])\n          this.searchPipeline.remove(lang[fn])\n        }\n\n      /* Set up index reference */\n      this.ref(\"location\")\n\n      /* Set up index fields */\n      this.field(\"title\", { boost: 1e3, extractor: field(\"title\") })\n      this.field(\"text\",  { boost: 1e0, extractor: field(\"text\") })\n      this.field(\"tags\",  { boost: 1e6, extractor: field(\"tags\") })\n\n      /* Add documents to index */\n      for (const doc of docs)\n        this.add(doc, { boost: doc.boost })\n    })\n  }\n\n  /**\n   * Search for matching documents\n   *\n   * @param query - Search query\n   *\n   * @returns Search result\n   */\n  public search(query: string): SearchResult {\n\n    // Experimental Chinese segmentation\n    query = query.replace(/\\p{sc=Han}+/gu, value => {\n      return [...segment(value, this.index.invertedIndex)]\n        .join(\"* \")\n    })\n\n    // @todo: move segmenter (above) into transformSearchQuery\n    query = transformSearchQuery(query)\n    if (!query)\n      return { items: [] }\n\n    /* Parse query to extract clauses for analysis */\n    const clauses = parseSearchQuery(query)\n      .filter(clause => (\n        clause.presence !== lunr.Query.presence.PROHIBITED\n      ))\n\n    /* Perform search and post-process results */\n    const groups = this.index.search(query)\n\n      /* Apply post-query boosts based on title and search query terms */\n      .reduce<SearchItem[]>((item, { ref, score, matchData }) => {\n        let doc = this.map.get(ref)\n        if (typeof doc !== \"undefined\") {\n\n          /* Shallow copy document */\n          doc = { ...doc }\n          if (doc.tags)\n            doc.tags = [...doc.tags]\n\n          /* Compute and analyze search query terms */\n          const terms = getSearchQueryTerms(\n            clauses,\n            Object.keys(matchData.metadata)\n          )\n\n          /* Highlight matches in fields */\n          for (const field of this.index.fields) {\n            if (typeof doc[field] === \"undefined\")\n              continue\n\n            /* Collect positions from matches */\n            const positions: Position[] = []\n            for (const match of Object.values(matchData.metadata))\n              if (typeof match[field] !== \"undefined\")\n                positions.push(...match[field].position)\n\n            /* Skip highlighting, if no positions were collected */\n            if (!positions.length)\n              continue\n\n            /* Load table and determine highlighting method */\n            const table = this.table.get([doc.location, field].join(\":\"))!\n            const fn = Array.isArray(doc[field])\n              ? highlightAll\n              : highlight\n\n            // @ts-expect-error - stop moaning, TypeScript!\n            doc[field] = fn(doc[field], table, positions, field !== \"text\")\n          }\n\n          /* Highlight title and text and apply post-query boosts */\n          const boost = +!doc.parent +\n            Object.values(terms)\n              .filter(t => t).length /\n            Object.keys(terms).length\n\n          /* Append item */\n          item.push({\n            ...doc,\n            score: score * (1 + boost ** 2),\n            terms\n          })\n        }\n        return item\n      }, [])\n\n      /* Sort search results again after applying boosts */\n      .sort((a, b) => b.score - a.score)\n\n      /* Group search results by article */\n      .reduce((items, result) => {\n        const doc = this.map.get(result.location)\n        if (typeof doc !== \"undefined\") {\n          const ref = doc.parent\n            ? doc.parent.location\n            : doc.location\n          items.set(ref, [...items.get(ref) || [], result])\n        }\n        return items\n      }, new Map<string, SearchItem[]>())\n\n    /* Ensure that every item set has an article */\n    for (const [ref, items] of groups)\n      if (!items.find(item => item.location === ref)) {\n        const doc = this.map.get(ref)!\n        items.push({ ...doc, score: 0, terms: {} })\n      }\n\n    /* Generate search suggestions, if desired */\n    let suggest: string[] | undefined\n    if (this.options.suggest) {\n      const titles = this.index.query(builder => {\n        for (const clause of clauses)\n          builder.term(clause.term, {\n            fields: [\"title\"],\n            presence: lunr.Query.presence.REQUIRED,\n            wildcard: lunr.Query.wildcard.TRAILING\n          })\n      })\n\n      /* Retrieve suggestions for best match */\n      suggest = titles.length\n        ? Object.keys(titles[0].matchData.metadata)\n        : []\n    }\n\n    /* Return search result */\n    return {\n      items: [...groups.values()],\n      ...typeof suggest !== \"undefined\" && { suggest }\n    }\n  }\n}\n"],
+  "mappings": "6lCAAA,IAAAA,GAAAC,GAAA,CAAAC,EAAAC,KAAA;AAAA;AAAA;AAAA;AAAA,IAME,UAAU,CAiCZ,IAAIC,EAAO,SAAUC,EAAQ,CAC3B,IAAIC,EAAU,IAAIF,EAAK,QAEvB,OAAAE,EAAQ,SAAS,IACfF,EAAK,QACLA,EAAK,eACLA,EAAK,OACP,EAEAE,EAAQ,eAAe,IACrBF,EAAK,OACP,EAEAC,EAAO,KAAKC,EAASA,CAAO,EACrBA,EAAQ,MAAM,CACvB,EAEAF,EAAK,QAAU,QACf;AAAA;AAAA;AAAA,GASAA,EAAK,MAAQ,CAAC,EASdA,EAAK,MAAM,KAAQ,SAAUG,EAAQ,CAEnC,OAAO,SAAUC,EAAS,CACpBD,EAAO,SAAW,QAAQ,MAC5B,QAAQ,KAAKC,CAAO,CAExB,CAEF,EAAG,IAAI,EAaPJ,EAAK,MAAM,SAAW,SAAUK,EAAK,CACnC,OAAsBA,GAAQ,KACrB,GAEAA,EAAI,SAAS,CAExB,EAkBAL,EAAK,MAAM,MAAQ,SAAUK,EAAK,CAChC,GAAIA,GAAQ,KACV,OAAOA,EAMT,QAHIC,EAAQ,OAAO,OAAO,IAAI,EAC1BC,EAAO,OAAO,KAAKF,CAAG,EAEjB,EAAI,EAAG,EAAIE,EAAK,OAAQ,IAAK,CACpC,IAAIC,EAAMD,EAAK,CAAC,EACZE,EAAMJ,EAAIG,CAAG,EAEjB,GAAI,MAAM,QAAQC,CAAG,EAAG,CACtBH,EAAME,CAAG,EAAIC,EAAI,MAAM,EACvB,QACF,CAEA,GAAI,OAAOA,GAAQ,UACf,OAAOA,GAAQ,UACf,OAAOA,GAAQ,UAAW,CAC5BH,EAAME,CAAG,EAAIC,EACb,QACF,CAEA,MAAM,IAAI,UAAU,uDAAuD,CAC7E,CAEA,OAAOH,CACT,EACAN,EAAK,SAAW,SAAUU,EAAQC,EAAWC,EAAa,CACxD,KAAK,OAASF,EACd,KAAK,UAAYC,EACjB,KAAK,aAAeC,CACtB,EAEAZ,EAAK,SAAS,OAAS,IAEvBA,EAAK,SAAS,WAAa,SAAUa,EAAG,CACtC,IAAIC,EAAID,EAAE,QAAQb,EAAK,SAAS,MAAM,EAEtC,GAAIc,IAAM,GACR,KAAM,6BAGR,IAAIC,EAAWF,EAAE,MAAM,EAAGC,CAAC,EACvBJ,EAASG,EAAE,MAAMC,EAAI,CAAC,EAE1B,OAAO,IAAId,EAAK,SAAUU,EAAQK,EAAUF,CAAC,CAC/C,EAEAb,EAAK,SAAS,UAAU,SAAW,UAAY,CAC7C,OAAI,KAAK,cAAgB,OACvB,KAAK,aAAe,KAAK,UAAYA,EAAK,SAAS,OAAS,KAAK,QAG5D,KAAK,YACd,EACA;AAAA;AAAA;AAAA,GAUAA,EAAK,IAAM,SAAUgB,EAAU,CAG7B,GAFA,KAAK,SAAW,OAAO,OAAO,IAAI,EAE9BA,EAAU,CACZ,KAAK,OAASA,EAAS,OAEvB,QAASC,EAAI,EAAGA,EAAI,KAAK,OAAQA,IAC/B,KAAK,SAASD,EAASC,CAAC,CAAC,EAAI,EAEjC,MACE,KAAK,OAAS,CAElB,EASAjB,EAAK,IAAI,SAAW,CAClB,UAAW,SAAUkB,EAAO,CAC1B,OAAOA,CACT,EAEA,MAAO,UAAY,CACjB,OAAO,IACT,EAEA,SAAU,UAAY,CACpB,MAAO,EACT,CACF,EASAlB,EAAK,IAAI,MAAQ,CACf,UAAW,UAAY,CACrB,OAAO,IACT,EAEA,MAAO,SAAUkB,EAAO,CACtB,OAAOA,CACT,EAEA,SAAU,UAAY,CACpB,MAAO,EACT,CACF,EAQAlB,EAAK,IAAI,UAAU,SAAW,SAAUmB,EAAQ,CAC9C,MAAO,CAAC,CAAC,KAAK,SAASA,CAAM,CAC/B,EAUAnB,EAAK,IAAI,UAAU,UAAY,SAAUkB,EAAO,CAC9C,IAAIE,EAAGC,EAAGL,EAAUM,EAAe,CAAC,EAEpC,GAAIJ,IAAUlB,EAAK,IAAI,SACrB,OAAO,KAGT,GAAIkB,IAAUlB,EAAK,IAAI,MACrB,OAAOkB,EAGL,KAAK,OAASA,EAAM,QACtBE,EAAI,KACJC,EAAIH,IAEJE,EAAIF,EACJG,EAAI,MAGNL,EAAW,OAAO,KAAKI,EAAE,QAAQ,EAEjC,QAASH,EAAI,EAAGA,EAAID,EAAS,OAAQC,IAAK,CACxC,IAAIM,EAAUP,EAASC,CAAC,EACpBM,KAAWF,EAAE,UACfC,EAAa,KAAKC,CAAO,CAE7B,CAEA,OAAO,IAAIvB,EAAK,IAAKsB,CAAY,CACnC,EASAtB,EAAK,IAAI,UAAU,MAAQ,SAAUkB,EAAO,CAC1C,OAAIA,IAAUlB,EAAK,IAAI,SACdA,EAAK,IAAI,SAGdkB,IAAUlB,EAAK,IAAI,MACd,KAGF,IAAIA,EAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,OAAO,OAAO,KAAKkB,EAAM,QAAQ,CAAC,CAAC,CACpF,EASAlB,EAAK,IAAM,SAAUwB,EAASC,EAAe,CAC3C,IAAIC,EAAoB,EAExB,QAASf,KAAaa,EAChBb,GAAa,WACjBe,GAAqB,OAAO,KAAKF,EAAQb,CAAS,CAAC,EAAE,QAGvD,IAAIgB,GAAKF,EAAgBC,EAAoB,KAAQA,EAAoB,IAEzE,OAAO,KAAK,IAAI,EAAI,KAAK,IAAIC,CAAC,CAAC,CACjC,EAUA3B,EAAK,MAAQ,SAAU4B,EAAKC,EAAU,CACpC,KAAK,IAAMD,GAAO,GAClB,KAAK,SAAWC,GAAY,CAAC,CAC/B,EAOA7B,EAAK,MAAM,UAAU,SAAW,UAAY,CAC1C,OAAO,KAAK,GACd,EAsBAA,EAAK,MAAM,UAAU,OAAS,SAAU8B,EAAI,CAC1C,YAAK,IAAMA,EAAG,KAAK,IAAK,KAAK,QAAQ,EAC9B,IACT,EASA9B,EAAK,MAAM,UAAU,MAAQ,SAAU8B,EAAI,CACzC,OAAAA,EAAKA,GAAM,SAAUjB,EAAG,CAAE,OAAOA,CAAE,EAC5B,IAAIb,EAAK,MAAO8B,EAAG,KAAK,IAAK,KAAK,QAAQ,EAAG,KAAK,QAAQ,CACnE,EACA;AAAA;AAAA;AAAA,GAuBA9B,EAAK,UAAY,SAAUK,EAAKwB,EAAU,CACxC,GAAIxB,GAAO,MAAQA,GAAO,KACxB,MAAO,CAAC,EAGV,GAAI,MAAM,QAAQA,CAAG,EACnB,OAAOA,EAAI,IAAI,SAAU0B,EAAG,CAC1B,OAAO,IAAI/B,EAAK,MACdA,EAAK,MAAM,SAAS+B,CAAC,EAAE,YAAY,EACnC/B,EAAK,MAAM,MAAM6B,CAAQ,CAC3B,CACF,CAAC,EAOH,QAJID,EAAMvB,EAAI,SAAS,EAAE,YAAY,EACjC2B,EAAMJ,EAAI,OACVK,EAAS,CAAC,EAELC,EAAW,EAAGC,EAAa,EAAGD,GAAYF,EAAKE,IAAY,CAClE,IAAIE,EAAOR,EAAI,OAAOM,CAAQ,EAC1BG,EAAcH,EAAWC,EAE7B,GAAKC,EAAK,MAAMpC,EAAK,UAAU,SAAS,GAAKkC,GAAYF,EAAM,CAE7D,GAAIK,EAAc,EAAG,CACnB,IAAIC,EAAgBtC,EAAK,MAAM,MAAM6B,CAAQ,GAAK,CAAC,EACnDS,EAAc,SAAc,CAACH,EAAYE,CAAW,EACpDC,EAAc,MAAWL,EAAO,OAEhCA,EAAO,KACL,IAAIjC,EAAK,MACP4B,EAAI,MAAMO,EAAYD,CAAQ,EAC9BI,CACF,CACF,CACF,CAEAH,EAAaD,EAAW,CAC1B,CAEF,CAEA,OAAOD,CACT,EASAjC,EAAK,UAAU,UAAY,UAC3B;AAAA;AAAA;AAAA,GAkCAA,EAAK,SAAW,UAAY,CAC1B,KAAK,OAAS,CAAC,CACjB,EAEAA,EAAK,SAAS,oBAAsB,OAAO,OAAO,IAAI,EAmCtDA,EAAK,SAAS,iBAAmB,SAAU8B,EAAIS,EAAO,CAChDA,KAAS,KAAK,qBAChBvC,EAAK,MAAM,KAAK,6CAA+CuC,CAAK,EAGtET,EAAG,MAAQS,EACXvC,EAAK,SAAS,oBAAoB8B,EAAG,KAAK,EAAIA,CAChD,EAQA9B,EAAK,SAAS,4BAA8B,SAAU8B,EAAI,CACxD,IAAIU,EAAeV,EAAG,OAAUA,EAAG,SAAS,KAAK,oBAE5CU,GACHxC,EAAK,MAAM,KAAK;AAAA,EAAmG8B,CAAE,CAEzH,EAYA9B,EAAK,SAAS,KAAO,SAAUyC,EAAY,CACzC,IAAIC,EAAW,IAAI1C,EAAK,SAExB,OAAAyC,EAAW,QAAQ,SAAUE,EAAQ,CACnC,IAAIb,EAAK9B,EAAK,SAAS,oBAAoB2C,CAAM,EAEjD,GAAIb,EACFY,EAAS,IAAIZ,CAAE,MAEf,OAAM,IAAI,MAAM,sCAAwCa,CAAM,CAElE,CAAC,EAEMD,CACT,EASA1C,EAAK,SAAS,UAAU,IAAM,UAAY,CACxC,IAAI4C,EAAM,MAAM,UAAU,MAAM,KAAK,SAAS,EAE9CA,EAAI,QAAQ,SAAUd,EAAI,CACxB9B,EAAK,SAAS,4BAA4B8B,CAAE,EAC5C,KAAK,OAAO,KAAKA,CAAE,CACrB,EAAG,IAAI,CACT,EAWA9B,EAAK,SAAS,UAAU,MAAQ,SAAU6C,EAAYC,EAAO,CAC3D9C,EAAK,SAAS,4BAA4B8C,CAAK,EAE/C,IAAIC,EAAM,KAAK,OAAO,QAAQF,CAAU,EACxC,GAAIE,GAAO,GACT,MAAM,IAAI,MAAM,wBAAwB,EAG1CA,EAAMA,EAAM,EACZ,KAAK,OAAO,OAAOA,EAAK,EAAGD,CAAK,CAClC,EAWA9C,EAAK,SAAS,UAAU,OAAS,SAAU6C,EAAYC,EAAO,CAC5D9C,EAAK,SAAS,4BAA4B8C,CAAK,EAE/C,IAAIC,EAAM,KAAK,OAAO,QAAQF,CAAU,EACxC,GAAIE,GAAO,GACT,MAAM,IAAI,MAAM,wBAAwB,EAG1C,KAAK,OAAO,OAAOA,EAAK,EAAGD,CAAK,CAClC,EAOA9C,EAAK,SAAS,UAAU,OAAS,SAAU8B,EAAI,CAC7C,IAAIiB,EAAM,KAAK,OAAO,QAAQjB,CAAE,EAC5BiB,GAAO,IAIX,KAAK,OAAO,OAAOA,EAAK,CAAC,CAC3B,EASA/C,EAAK,SAAS,UAAU,IAAM,SAAUiC,EAAQ,CAG9C,QAFIe,EAAc,KAAK,OAAO,OAErB/B,EAAI,EAAGA,EAAI+B,EAAa/B,IAAK,CAIpC,QAHIa,EAAK,KAAK,OAAOb,CAAC,EAClBgC,EAAO,CAAC,EAEHC,EAAI,EAAGA,EAAIjB,EAAO,OAAQiB,IAAK,CACtC,IAAIC,EAASrB,EAAGG,EAAOiB,CAAC,EAAGA,EAAGjB,CAAM,EAEpC,GAAI,EAAAkB,GAAW,MAA6BA,IAAW,IAEvD,GAAI,MAAM,QAAQA,CAAM,EACtB,QAASC,EAAI,EAAGA,EAAID,EAAO,OAAQC,IACjCH,EAAK,KAAKE,EAAOC,CAAC,CAAC,OAGrBH,EAAK,KAAKE,CAAM,CAEpB,CAEAlB,EAASgB,CACX,CAEA,OAAOhB,CACT,EAYAjC,EAAK,SAAS,UAAU,UAAY,SAAU4B,EAAKC,EAAU,CAC3D,IAAIwB,EAAQ,IAAIrD,EAAK,MAAO4B,EAAKC,CAAQ,EAEzC,OAAO,KAAK,IAAI,CAACwB,CAAK,CAAC,EAAE,IAAI,SAAUtB,EAAG,CACxC,OAAOA,EAAE,SAAS,CACpB,CAAC,CACH,EAMA/B,EAAK,SAAS,UAAU,MAAQ,UAAY,CAC1C,KAAK,OAAS,CAAC,CACjB,EASAA,EAAK,SAAS,UAAU,OAAS,UAAY,CAC3C,OAAO,KAAK,OAAO,IAAI,SAAU8B,EAAI,CACnC,OAAA9B,EAAK,SAAS,4BAA4B8B,CAAE,EAErCA,EAAG,KACZ,CAAC,CACH,EACA;AAAA;AAAA;AAAA,GAqBA9B,EAAK,OAAS,SAAUgB,EAAU,CAChC,KAAK,WAAa,EAClB,KAAK,SAAWA,GAAY,CAAC,CAC/B,EAaAhB,EAAK,OAAO,UAAU,iBAAmB,SAAUsD,EAAO,CAExD,GAAI,KAAK,SAAS,QAAU,EAC1B,MAAO,GAST,QANIC,EAAQ,EACRC,EAAM,KAAK,SAAS,OAAS,EAC7BnB,EAAcmB,EAAMD,EACpBE,EAAa,KAAK,MAAMpB,EAAc,CAAC,EACvCqB,EAAa,KAAK,SAASD,EAAa,CAAC,EAEtCpB,EAAc,IACfqB,EAAaJ,IACfC,EAAQE,GAGNC,EAAaJ,IACfE,EAAMC,GAGJC,GAAcJ,IAIlBjB,EAAcmB,EAAMD,EACpBE,EAAaF,EAAQ,KAAK,MAAMlB,EAAc,CAAC,EAC/CqB,EAAa,KAAK,SAASD,EAAa,CAAC,EAO3C,GAJIC,GAAcJ,GAIdI,EAAaJ,EACf,OAAOG,EAAa,EAGtB,GAAIC,EAAaJ,EACf,OAAQG,EAAa,GAAK,CAE9B,EAWAzD,EAAK,OAAO,UAAU,OAAS,SAAU2D,EAAWlD,EAAK,CACvD,KAAK,OAAOkD,EAAWlD,EAAK,UAAY,CACtC,KAAM,iBACR,CAAC,CACH,EAUAT,EAAK,OAAO,UAAU,OAAS,SAAU2D,EAAWlD,EAAKqB,EAAI,CAC3D,KAAK,WAAa,EAClB,IAAI8B,EAAW,KAAK,iBAAiBD,CAAS,EAE1C,KAAK,SAASC,CAAQ,GAAKD,EAC7B,KAAK,SAASC,EAAW,CAAC,EAAI9B,EAAG,KAAK,SAAS8B,EAAW,CAAC,EAAGnD,CAAG,EAEjE,KAAK,SAAS,OAAOmD,EAAU,EAAGD,EAAWlD,CAAG,CAEpD,EAOAT,EAAK,OAAO,UAAU,UAAY,UAAY,CAC5C,GAAI,KAAK,WAAY,OAAO,KAAK,WAKjC,QAHI6D,EAAe,EACfC,EAAiB,KAAK,SAAS,OAE1B7C,EAAI,EAAGA,EAAI6C,EAAgB7C,GAAK,EAAG,CAC1C,IAAIR,EAAM,KAAK,SAASQ,CAAC,EACzB4C,GAAgBpD,EAAMA,CACxB,CAEA,OAAO,KAAK,WAAa,KAAK,KAAKoD,CAAY,CACjD,EAQA7D,EAAK,OAAO,UAAU,IAAM,SAAU+D,EAAa,CAOjD,QANIC,EAAa,EACb5C,EAAI,KAAK,SAAUC,EAAI0C,EAAY,SACnCE,EAAO7C,EAAE,OAAQ8C,EAAO7C,EAAE,OAC1B8C,EAAO,EAAGC,EAAO,EACjBnD,EAAI,EAAGiC,EAAI,EAERjC,EAAIgD,GAAQf,EAAIgB,GACrBC,EAAO/C,EAAEH,CAAC,EAAGmD,EAAO/C,EAAE6B,CAAC,EACnBiB,EAAOC,EACTnD,GAAK,EACIkD,EAAOC,EAChBlB,GAAK,EACIiB,GAAQC,IACjBJ,GAAc5C,EAAEH,EAAI,CAAC,EAAII,EAAE6B,EAAI,CAAC,EAChCjC,GAAK,EACLiC,GAAK,GAIT,OAAOc,CACT,EASAhE,EAAK,OAAO,UAAU,WAAa,SAAU+D,EAAa,CACxD,OAAO,KAAK,IAAIA,CAAW,EAAI,KAAK,UAAU,GAAK,CACrD,EAOA/D,EAAK,OAAO,UAAU,QAAU,UAAY,CAG1C,QAFIqE,EAAS,IAAI,MAAO,KAAK,SAAS,OAAS,CAAC,EAEvCpD,EAAI,EAAGiC,EAAI,EAAGjC,EAAI,KAAK,SAAS,OAAQA,GAAK,EAAGiC,IACvDmB,EAAOnB,CAAC,EAAI,KAAK,SAASjC,CAAC,EAG7B,OAAOoD,CACT,EAOArE,EAAK,OAAO,UAAU,OAAS,UAAY,CACzC,OAAO,KAAK,QACd,EAEA;AAAA;AAAA;AAAA;AAAA,GAiBAA,EAAK,QAAW,UAAU,CACxB,IAAIsE,EAAY,CACZ,QAAY,MACZ,OAAW,OACX,KAAS,OACT,KAAS,OACT,KAAS,MACT,IAAQ,MACR,KAAS,KACT,MAAU,MACV,IAAQ,IACR,MAAU,MACV,QAAY,MACZ,MAAU,MACV,KAAS,MACT,MAAU,KACV,QAAY,MACZ,QAAY,MACZ,QAAY,MACZ,MAAU,KACV,MAAU,MACV,OAAW,MACX,KAAS,KACX,EAEAC,EAAY,CACV,MAAU,KACV,MAAU,GACV,MAAU,KACV,MAAU,KACV,KAAS,KACT,IAAQ,GACR,KAAS,EACX,EAEAC,EAAI,WACJC,EAAI,WACJC,EAAIF,EAAI,aACRG,EAAIF,EAAI,WAERG,EAAO,KAAOF,EAAI,KAAOC,EAAID,EAC7BG,EAAO,KAAOH,EAAI,KAAOC,EAAID,EAAI,IAAMC,EAAI,MAC3CG,EAAO,KAAOJ,EAAI,KAAOC,EAAID,EAAIC,EAAID,EACrCK,EAAM,KAAOL,EAAI,KAAOD,EAEtBO,EAAU,IAAI,OAAOJ,CAAI,EACzBK,EAAU,IAAI,OAAOH,CAAI,EACzBI,EAAU,IAAI,OAAOL,CAAI,EACzBM,EAAS,IAAI,OAAOJ,CAAG,EAEvBK,EAAQ,kBACRC,EAAS,iBACTC,EAAQ,aACRC,EAAS,kBACTC,EAAU,KACVC,EAAW,cACXC,EAAW,IAAI,OAAO,oBAAoB,EAC1CC,EAAW,IAAI,OAAO,IAAMjB,EAAID,EAAI,cAAc,EAElDmB,EAAQ,mBACRC,EAAO,2IAEPC,EAAO,iDAEPC,EAAO,sFACPC,EAAQ,oBAERC,EAAO,WACPC,EAAS,MACTC,EAAQ,IAAI,OAAO,IAAMzB,EAAID,EAAI,cAAc,EAE/C2B,EAAgB,SAAuBC,EAAG,CAC5C,IAAIC,EACFC,EACAC,EACAC,EACAC,EACAC,EACAC,EAEF,GAAIP,EAAE,OAAS,EAAK,OAAOA,EAiB3B,GAfAG,EAAUH,EAAE,OAAO,EAAE,CAAC,EAClBG,GAAW,MACbH,EAAIG,EAAQ,YAAY,EAAIH,EAAE,OAAO,CAAC,GAIxCI,EAAKrB,EACLsB,EAAMrB,EAEFoB,EAAG,KAAKJ,CAAC,EAAKA,EAAIA,EAAE,QAAQI,EAAG,MAAM,EAChCC,EAAI,KAAKL,CAAC,IAAKA,EAAIA,EAAE,QAAQK,EAAI,MAAM,GAGhDD,EAAKnB,EACLoB,EAAMnB,EACFkB,EAAG,KAAKJ,CAAC,EAAG,CACd,IAAIQ,EAAKJ,EAAG,KAAKJ,CAAC,EAClBI,EAAKzB,EACDyB,EAAG,KAAKI,EAAG,CAAC,CAAC,IACfJ,EAAKjB,EACLa,EAAIA,EAAE,QAAQI,EAAG,EAAE,EAEvB,SAAWC,EAAI,KAAKL,CAAC,EAAG,CACtB,IAAIQ,EAAKH,EAAI,KAAKL,CAAC,EACnBC,EAAOO,EAAG,CAAC,EACXH,EAAMvB,EACFuB,EAAI,KAAKJ,CAAI,IACfD,EAAIC,EACJI,EAAMjB,EACNkB,EAAMjB,EACNkB,EAAMjB,EACFe,EAAI,KAAKL,CAAC,EAAKA,EAAIA,EAAI,IAClBM,EAAI,KAAKN,CAAC,GAAKI,EAAKjB,EAASa,EAAIA,EAAE,QAAQI,EAAG,EAAE,GAChDG,EAAI,KAAKP,CAAC,IAAKA,EAAIA,EAAI,KAEpC,CAIA,GADAI,EAAKb,EACDa,EAAG,KAAKJ,CAAC,EAAG,CACd,IAAIQ,EAAKJ,EAAG,KAAKJ,CAAC,EAClBC,EAAOO,EAAG,CAAC,EACXR,EAAIC,EAAO,GACb,CAIA,GADAG,EAAKZ,EACDY,EAAG,KAAKJ,CAAC,EAAG,CACd,IAAIQ,EAAKJ,EAAG,KAAKJ,CAAC,EAClBC,EAAOO,EAAG,CAAC,EACXN,EAASM,EAAG,CAAC,EACbJ,EAAKzB,EACDyB,EAAG,KAAKH,CAAI,IACdD,EAAIC,EAAOhC,EAAUiC,CAAM,EAE/B,CAIA,GADAE,EAAKX,EACDW,EAAG,KAAKJ,CAAC,EAAG,CACd,IAAIQ,EAAKJ,EAAG,KAAKJ,CAAC,EAClBC,EAAOO,EAAG,CAAC,EACXN,EAASM,EAAG,CAAC,EACbJ,EAAKzB,EACDyB,EAAG,KAAKH,CAAI,IACdD,EAAIC,EAAO/B,EAAUgC,CAAM,EAE/B,CAKA,GAFAE,EAAKV,EACLW,EAAMV,EACFS,EAAG,KAAKJ,CAAC,EAAG,CACd,IAAIQ,EAAKJ,EAAG,KAAKJ,CAAC,EAClBC,EAAOO,EAAG,CAAC,EACXJ,EAAKxB,EACDwB,EAAG,KAAKH,CAAI,IACdD,EAAIC,EAER,SAAWI,EAAI,KAAKL,CAAC,EAAG,CACtB,IAAIQ,EAAKH,EAAI,KAAKL,CAAC,EACnBC,EAAOO,EAAG,CAAC,EAAIA,EAAG,CAAC,EACnBH,EAAMzB,EACFyB,EAAI,KAAKJ,CAAI,IACfD,EAAIC,EAER,CAIA,GADAG,EAAKR,EACDQ,EAAG,KAAKJ,CAAC,EAAG,CACd,IAAIQ,EAAKJ,EAAG,KAAKJ,CAAC,EAClBC,EAAOO,EAAG,CAAC,EACXJ,EAAKxB,EACLyB,EAAMxB,EACNyB,EAAMR,GACFM,EAAG,KAAKH,CAAI,GAAMI,EAAI,KAAKJ,CAAI,GAAK,CAAEK,EAAI,KAAKL,CAAI,KACrDD,EAAIC,EAER,CAEA,OAAAG,EAAKP,EACLQ,EAAMzB,EACFwB,EAAG,KAAKJ,CAAC,GAAKK,EAAI,KAAKL,CAAC,IAC1BI,EAAKjB,EACLa,EAAIA,EAAE,QAAQI,EAAG,EAAE,GAKjBD,GAAW,MACbH,EAAIG,EAAQ,YAAY,EAAIH,EAAE,OAAO,CAAC,GAGjCA,CACT,EAEA,OAAO,SAAUhD,EAAO,CACtB,OAAOA,EAAM,OAAO+C,CAAa,CACnC,CACF,EAAG,EAEHpG,EAAK,SAAS,iBAAiBA,EAAK,QAAS,SAAS,EACtD;AAAA;AAAA;AAAA,GAkBAA,EAAK,uBAAyB,SAAU8G,EAAW,CACjD,IAAIC,EAAQD,EAAU,OAAO,SAAU7D,EAAM+D,EAAU,CACrD,OAAA/D,EAAK+D,CAAQ,EAAIA,EACV/D,CACT,EAAG,CAAC,CAAC,EAEL,OAAO,SAAUI,EAAO,CACtB,GAAIA,GAAS0D,EAAM1D,EAAM,SAAS,CAAC,IAAMA,EAAM,SAAS,EAAG,OAAOA,CACpE,CACF,EAeArD,EAAK,eAAiBA,EAAK,uBAAuB,CAChD,IACA,OACA,QACA,SACA,QACA,MACA,SACA,OACA,KACA,QACA,KACA,MACA,MACA,MACA,KACA,KACA,KACA,UACA,OACA,MACA,KACA,MACA,SACA,QACA,OACA,MACA,KACA,OACA,SACA,OACA,OACA,QACA,MACA,OACA,MACA,MACA,MACA,MACA,OACA,KACA,MACA,OACA,MACA,MACA,MACA,UACA,IACA,KACA,KACA,OACA,KACA,KACA,MACA,OACA,QACA,MACA,OACA,SACA,MACA,KACA,QACA,OACA,OACA,KACA,UACA,KACA,MACA,MACA,KACA,MACA,QACA,KACA,OACA,KACA,QACA,MACA,MACA,SACA,OACA,MACA,OACA,MACA,SACA,QACA,KACA,OACA,OACA,OACA,MACA,QACA,OACA,OACA,QACA,QACA,OACA,OACA,MACA,KACA,MACA,OACA,KACA,QACA,MACA,KACA,OACA,OACA,OACA,QACA,QACA,QACA,MACA,OACA,MACA,OACA,OACA,QACA,MACA,MACA,MACF,CAAC,EAEDA,EAAK,SAAS,iBAAiBA,EAAK,eAAgB,gBAAgB,EACpE;AAAA;AAAA;AAAA,GAoBAA,EAAK,QAAU,SAAUqD,EAAO,CAC9B,OAAOA,EAAM,OAAO,SAAUxC,EAAG,CAC/B,OAAOA,EAAE,QAAQ,OAAQ,EAAE,EAAE,QAAQ,OAAQ,EAAE,CACjD,CAAC,CACH,EAEAb,EAAK,SAAS,iBAAiBA,EAAK,QAAS,SAAS,EACtD;AAAA;AAAA;AAAA,GA0BAA,EAAK,SAAW,UAAY,CAC1B,KAAK,MAAQ,GACb,KAAK,MAAQ,CAAC,EACd,KAAK,GAAKA,EAAK,SAAS,QACxBA,EAAK,SAAS,SAAW,CAC3B,EAUAA,EAAK,SAAS,QAAU,EASxBA,EAAK,SAAS,UAAY,SAAUiH,EAAK,CAGvC,QAFI/G,EAAU,IAAIF,EAAK,SAAS,QAEvBiB,EAAI,EAAGe,EAAMiF,EAAI,OAAQhG,EAAIe,EAAKf,IACzCf,EAAQ,OAAO+G,EAAIhG,CAAC,CAAC,EAGvB,OAAAf,EAAQ,OAAO,EACRA,EAAQ,IACjB,EAWAF,EAAK,SAAS,WAAa,SAAUkH,EAAQ,CAC3C,MAAI,iBAAkBA,EACblH,EAAK,SAAS,gBAAgBkH,EAAO,KAAMA,EAAO,YAAY,EAE9DlH,EAAK,SAAS,WAAWkH,EAAO,IAAI,CAE/C,EAiBAlH,EAAK,SAAS,gBAAkB,SAAU4B,EAAKuF,EAAc,CAS3D,QARIC,EAAO,IAAIpH,EAAK,SAEhBqH,EAAQ,CAAC,CACX,KAAMD,EACN,eAAgBD,EAChB,IAAKvF,CACP,CAAC,EAEMyF,EAAM,QAAQ,CACnB,IAAIC,EAAQD,EAAM,IAAI,EAGtB,GAAIC,EAAM,IAAI,OAAS,EAAG,CACxB,IAAIlF,EAAOkF,EAAM,IAAI,OAAO,CAAC,EACzBC,EAEAnF,KAAQkF,EAAM,KAAK,MACrBC,EAAaD,EAAM,KAAK,MAAMlF,CAAI,GAElCmF,EAAa,IAAIvH,EAAK,SACtBsH,EAAM,KAAK,MAAMlF,CAAI,EAAImF,GAGvBD,EAAM,IAAI,QAAU,IACtBC,EAAW,MAAQ,IAGrBF,EAAM,KAAK,CACT,KAAME,EACN,eAAgBD,EAAM,eACtB,IAAKA,EAAM,IAAI,MAAM,CAAC,CACxB,CAAC,CACH,CAEA,GAAIA,EAAM,gBAAkB,EAK5B,IAAI,MAAOA,EAAM,KAAK,MACpB,IAAIE,EAAgBF,EAAM,KAAK,MAAM,GAAG,MACnC,CACL,IAAIE,EAAgB,IAAIxH,EAAK,SAC7BsH,EAAM,KAAK,MAAM,GAAG,EAAIE,CAC1B,CAgCA,GA9BIF,EAAM,IAAI,QAAU,IACtBE,EAAc,MAAQ,IAGxBH,EAAM,KAAK,CACT,KAAMG,EACN,eAAgBF,EAAM,eAAiB,EACvC,IAAKA,EAAM,GACb,CAAC,EAKGA,EAAM,IAAI,OAAS,GACrBD,EAAM,KAAK,CACT,KAAMC,EAAM,KACZ,eAAgBA,EAAM,eAAiB,EACvC,IAAKA,EAAM,IAAI,MAAM,CAAC,CACxB,CAAC,EAKCA,EAAM,IAAI,QAAU,IACtBA,EAAM,KAAK,MAAQ,IAMjBA,EAAM,IAAI,QAAU,EAAG,CACzB,GAAI,MAAOA,EAAM,KAAK,MACpB,IAAIG,EAAmBH,EAAM,KAAK,MAAM,GAAG,MACtC,CACL,IAAIG,EAAmB,IAAIzH,EAAK,SAChCsH,EAAM,KAAK,MAAM,GAAG,EAAIG,CAC1B,CAEIH,EAAM,IAAI,QAAU,IACtBG,EAAiB,MAAQ,IAG3BJ,EAAM,KAAK,CACT,KAAMI,EACN,eAAgBH,EAAM,eAAiB,EACvC,IAAKA,EAAM,IAAI,MAAM,CAAC,CACxB,CAAC,CACH,CAKA,GAAIA,EAAM,IAAI,OAAS,EAAG,CACxB,IAAII,EAAQJ,EAAM,IAAI,OAAO,CAAC,EAC1BK,EAAQL,EAAM,IAAI,OAAO,CAAC,EAC1BM,EAEAD,KAASL,EAAM,KAAK,MACtBM,EAAgBN,EAAM,KAAK,MAAMK,CAAK,GAEtCC,EAAgB,IAAI5H,EAAK,SACzBsH,EAAM,KAAK,MAAMK,CAAK,EAAIC,GAGxBN,EAAM,IAAI,QAAU,IACtBM,EAAc,MAAQ,IAGxBP,EAAM,KAAK,CACT,KAAMO,EACN,eAAgBN,EAAM,eAAiB,EACvC,IAAKI,EAAQJ,EAAM,IAAI,MAAM,CAAC,CAChC,CAAC,CACH,EACF,CAEA,OAAOF,CACT,EAYApH,EAAK,SAAS,WAAa,SAAU4B,EAAK,CAYxC,QAXIiG,EAAO,IAAI7H,EAAK,SAChBoH,EAAOS,EAUF,EAAI,EAAG7F,EAAMJ,EAAI,OAAQ,EAAII,EAAK,IAAK,CAC9C,IAAII,EAAOR,EAAI,CAAC,EACZkG,EAAS,GAAK9F,EAAM,EAExB,GAAII,GAAQ,IACVyF,EAAK,MAAMzF,CAAI,EAAIyF,EACnBA,EAAK,MAAQC,MAER,CACL,IAAIC,EAAO,IAAI/H,EAAK,SACpB+H,EAAK,MAAQD,EAEbD,EAAK,MAAMzF,CAAI,EAAI2F,EACnBF,EAAOE,CACT,CACF,CAEA,OAAOX,CACT,EAYApH,EAAK,SAAS,UAAU,QAAU,UAAY,CAQ5C,QAPI+G,EAAQ,CAAC,EAETM,EAAQ,CAAC,CACX,OAAQ,GACR,KAAM,IACR,CAAC,EAEMA,EAAM,QAAQ,CACnB,IAAIC,EAAQD,EAAM,IAAI,EAClBW,EAAQ,OAAO,KAAKV,EAAM,KAAK,KAAK,EACpCtF,EAAMgG,EAAM,OAEZV,EAAM,KAAK,QAKbA,EAAM,OAAO,OAAO,CAAC,EACrBP,EAAM,KAAKO,EAAM,MAAM,GAGzB,QAASrG,EAAI,EAAGA,EAAIe,EAAKf,IAAK,CAC5B,IAAIgH,EAAOD,EAAM/G,CAAC,EAElBoG,EAAM,KAAK,CACT,OAAQC,EAAM,OAAO,OAAOW,CAAI,EAChC,KAAMX,EAAM,KAAK,MAAMW,CAAI,CAC7B,CAAC,CACH,CACF,CAEA,OAAOlB,CACT,EAYA/G,EAAK,SAAS,UAAU,SAAW,UAAY,CAS7C,GAAI,KAAK,KACP,OAAO,KAAK,KAOd,QAJI4B,EAAM,KAAK,MAAQ,IAAM,IACzBsG,EAAS,OAAO,KAAK,KAAK,KAAK,EAAE,KAAK,EACtClG,EAAMkG,EAAO,OAER,EAAI,EAAG,EAAIlG,EAAK,IAAK,CAC5B,IAAIO,EAAQ2F,EAAO,CAAC,EAChBL,EAAO,KAAK,MAAMtF,CAAK,EAE3BX,EAAMA,EAAMW,EAAQsF,EAAK,EAC3B,CAEA,OAAOjG,CACT,EAYA5B,EAAK,SAAS,UAAU,UAAY,SAAUqB,EAAG,CAU/C,QATIgD,EAAS,IAAIrE,EAAK,SAClBsH,EAAQ,OAERD,EAAQ,CAAC,CACX,MAAOhG,EACP,OAAQgD,EACR,KAAM,IACR,CAAC,EAEMgD,EAAM,QAAQ,CACnBC,EAAQD,EAAM,IAAI,EAWlB,QALIc,EAAS,OAAO,KAAKb,EAAM,MAAM,KAAK,EACtCc,EAAOD,EAAO,OACdE,EAAS,OAAO,KAAKf,EAAM,KAAK,KAAK,EACrCgB,EAAOD,EAAO,OAETE,EAAI,EAAGA,EAAIH,EAAMG,IAGxB,QAFIC,EAAQL,EAAOI,CAAC,EAEXzH,EAAI,EAAGA,EAAIwH,EAAMxH,IAAK,CAC7B,IAAI2H,EAAQJ,EAAOvH,CAAC,EAEpB,GAAI2H,GAASD,GAASA,GAAS,IAAK,CAClC,IAAIX,EAAOP,EAAM,KAAK,MAAMmB,CAAK,EAC7BC,EAAQpB,EAAM,MAAM,MAAMkB,CAAK,EAC/BV,EAAQD,EAAK,OAASa,EAAM,MAC5BX,EAAO,OAEPU,KAASnB,EAAM,OAAO,OAIxBS,EAAOT,EAAM,OAAO,MAAMmB,CAAK,EAC/BV,EAAK,MAAQA,EAAK,OAASD,IAM3BC,EAAO,IAAI/H,EAAK,SAChB+H,EAAK,MAAQD,EACbR,EAAM,OAAO,MAAMmB,CAAK,EAAIV,GAG9BV,EAAM,KAAK,CACT,MAAOqB,EACP,OAAQX,EACR,KAAMF,CACR,CAAC,CACH,CACF,CAEJ,CAEA,OAAOxD,CACT,EACArE,EAAK,SAAS,QAAU,UAAY,CAClC,KAAK,aAAe,GACpB,KAAK,KAAO,IAAIA,EAAK,SACrB,KAAK,eAAiB,CAAC,EACvB,KAAK,eAAiB,CAAC,CACzB,EAEAA,EAAK,SAAS,QAAQ,UAAU,OAAS,SAAU2I,EAAM,CACvD,IAAId,EACAe,EAAe,EAEnB,GAAID,EAAO,KAAK,aACd,MAAM,IAAI,MAAO,6BAA6B,EAGhD,QAAS,EAAI,EAAG,EAAIA,EAAK,QAAU,EAAI,KAAK,aAAa,QACnDA,EAAK,CAAC,GAAK,KAAK,aAAa,CAAC,EAD6B,IAE/DC,IAGF,KAAK,SAASA,CAAY,EAEtB,KAAK,eAAe,QAAU,EAChCf,EAAO,KAAK,KAEZA,EAAO,KAAK,eAAe,KAAK,eAAe,OAAS,CAAC,EAAE,MAG7D,QAAS,EAAIe,EAAc,EAAID,EAAK,OAAQ,IAAK,CAC/C,IAAIE,EAAW,IAAI7I,EAAK,SACpBoC,EAAOuG,EAAK,CAAC,EAEjBd,EAAK,MAAMzF,CAAI,EAAIyG,EAEnB,KAAK,eAAe,KAAK,CACvB,OAAQhB,EACR,KAAMzF,EACN,MAAOyG,CACT,CAAC,EAEDhB,EAAOgB,CACT,CAEAhB,EAAK,MAAQ,GACb,KAAK,aAAec,CACtB,EAEA3I,EAAK,SAAS,QAAQ,UAAU,OAAS,UAAY,CACnD,KAAK,SAAS,CAAC,CACjB,EAEAA,EAAK,SAAS,QAAQ,UAAU,SAAW,SAAU8I,EAAQ,CAC3D,QAAS7H,EAAI,KAAK,eAAe,OAAS,EAAGA,GAAK6H,EAAQ7H,IAAK,CAC7D,IAAI4G,EAAO,KAAK,eAAe5G,CAAC,EAC5B8H,EAAWlB,EAAK,MAAM,SAAS,EAE/BkB,KAAY,KAAK,eACnBlB,EAAK,OAAO,MAAMA,EAAK,IAAI,EAAI,KAAK,eAAekB,CAAQ,GAI3DlB,EAAK,MAAM,KAAOkB,EAElB,KAAK,eAAeA,CAAQ,EAAIlB,EAAK,OAGvC,KAAK,eAAe,IAAI,CAC1B,CACF,EACA;AAAA;AAAA;AAAA,GAqBA7H,EAAK,MAAQ,SAAUgJ,EAAO,CAC5B,KAAK,cAAgBA,EAAM,cAC3B,KAAK,aAAeA,EAAM,aAC1B,KAAK,SAAWA,EAAM,SACtB,KAAK,OAASA,EAAM,OACpB,KAAK,SAAWA,EAAM,QACxB,EAyEAhJ,EAAK,MAAM,UAAU,OAAS,SAAUiJ,EAAa,CACnD,OAAO,KAAK,MAAM,SAAUC,EAAO,CACjC,IAAIC,EAAS,IAAInJ,EAAK,YAAYiJ,EAAaC,CAAK,EACpDC,EAAO,MAAM,CACf,CAAC,CACH,EA2BAnJ,EAAK,MAAM,UAAU,MAAQ,SAAU8B,EAAI,CAoBzC,QAZIoH,EAAQ,IAAIlJ,EAAK,MAAM,KAAK,MAAM,EAClCoJ,EAAiB,OAAO,OAAO,IAAI,EACnCC,EAAe,OAAO,OAAO,IAAI,EACjCC,EAAiB,OAAO,OAAO,IAAI,EACnCC,EAAkB,OAAO,OAAO,IAAI,EACpCC,EAAoB,OAAO,OAAO,IAAI,EAOjCvI,EAAI,EAAGA,EAAI,KAAK,OAAO,OAAQA,IACtCoI,EAAa,KAAK,OAAOpI,CAAC,CAAC,EAAI,IAAIjB,EAAK,OAG1C8B,EAAG,KAAKoH,EAAOA,CAAK,EAEpB,QAASjI,EAAI,EAAGA,EAAIiI,EAAM,QAAQ,OAAQjI,IAAK,CAS7C,IAAIiG,EAASgC,EAAM,QAAQjI,CAAC,EACxBwI,EAAQ,KACRC,EAAgB1J,EAAK,IAAI,MAEzBkH,EAAO,YACTuC,EAAQ,KAAK,SAAS,UAAUvC,EAAO,KAAM,CAC3C,OAAQA,EAAO,MACjB,CAAC,EAEDuC,EAAQ,CAACvC,EAAO,IAAI,EAGtB,QAASyC,EAAI,EAAGA,EAAIF,EAAM,OAAQE,IAAK,CACrC,IAAIC,EAAOH,EAAME,CAAC,EAQlBzC,EAAO,KAAO0C,EAOd,IAAIC,EAAe7J,EAAK,SAAS,WAAWkH,CAAM,EAC9C4C,EAAgB,KAAK,SAAS,UAAUD,CAAY,EAAE,QAAQ,EAQlE,GAAIC,EAAc,SAAW,GAAK5C,EAAO,WAAalH,EAAK,MAAM,SAAS,SAAU,CAClF,QAASoD,EAAI,EAAGA,EAAI8D,EAAO,OAAO,OAAQ9D,IAAK,CAC7C,IAAI2G,EAAQ7C,EAAO,OAAO9D,CAAC,EAC3BmG,EAAgBQ,CAAK,EAAI/J,EAAK,IAAI,KACpC,CAEA,KACF,CAEA,QAASkD,EAAI,EAAGA,EAAI4G,EAAc,OAAQ5G,IASxC,QAJI8G,EAAeF,EAAc5G,CAAC,EAC9B1B,EAAU,KAAK,cAAcwI,CAAY,EACzCC,EAAYzI,EAAQ,OAEf4B,EAAI,EAAGA,EAAI8D,EAAO,OAAO,OAAQ9D,IAAK,CAS7C,IAAI2G,EAAQ7C,EAAO,OAAO9D,CAAC,EACvB8G,EAAe1I,EAAQuI,CAAK,EAC5BI,EAAuB,OAAO,KAAKD,CAAY,EAC/CE,EAAYJ,EAAe,IAAMD,EACjCM,EAAuB,IAAIrK,EAAK,IAAImK,CAAoB,EAoB5D,GAbIjD,EAAO,UAAYlH,EAAK,MAAM,SAAS,WACzC0J,EAAgBA,EAAc,MAAMW,CAAoB,EAEpDd,EAAgBQ,CAAK,IAAM,SAC7BR,EAAgBQ,CAAK,EAAI/J,EAAK,IAAI,WASlCkH,EAAO,UAAYlH,EAAK,MAAM,SAAS,WAAY,CACjDwJ,EAAkBO,CAAK,IAAM,SAC/BP,EAAkBO,CAAK,EAAI/J,EAAK,IAAI,OAGtCwJ,EAAkBO,CAAK,EAAIP,EAAkBO,CAAK,EAAE,MAAMM,CAAoB,EAO9E,QACF,CAeA,GANAhB,EAAaU,CAAK,EAAE,OAAOE,EAAW/C,EAAO,MAAO,SAAU9F,GAAGC,GAAG,CAAE,OAAOD,GAAIC,EAAE,CAAC,EAMhF,CAAAiI,EAAec,CAAS,EAI5B,SAASE,EAAI,EAAGA,EAAIH,EAAqB,OAAQG,IAAK,CAOpD,IAAIC,EAAsBJ,EAAqBG,CAAC,EAC5CE,EAAmB,IAAIxK,EAAK,SAAUuK,EAAqBR,CAAK,EAChElI,EAAWqI,EAAaK,CAAmB,EAC3CE,GAECA,EAAarB,EAAeoB,CAAgB,KAAO,OACtDpB,EAAeoB,CAAgB,EAAI,IAAIxK,EAAK,UAAWgK,EAAcD,EAAOlI,CAAQ,EAEpF4I,EAAW,IAAIT,EAAcD,EAAOlI,CAAQ,CAGhD,CAEAyH,EAAec,CAAS,EAAI,GAC9B,CAEJ,CAQA,GAAIlD,EAAO,WAAalH,EAAK,MAAM,SAAS,SAC1C,QAASoD,EAAI,EAAGA,EAAI8D,EAAO,OAAO,OAAQ9D,IAAK,CAC7C,IAAI2G,EAAQ7C,EAAO,OAAO9D,CAAC,EAC3BmG,EAAgBQ,CAAK,EAAIR,EAAgBQ,CAAK,EAAE,UAAUL,CAAa,CACzE,CAEJ,CAUA,QAHIgB,EAAqB1K,EAAK,IAAI,SAC9B2K,EAAuB3K,EAAK,IAAI,MAE3BiB,EAAI,EAAGA,EAAI,KAAK,OAAO,OAAQA,IAAK,CAC3C,IAAI8I,EAAQ,KAAK,OAAO9I,CAAC,EAErBsI,EAAgBQ,CAAK,IACvBW,EAAqBA,EAAmB,UAAUnB,EAAgBQ,CAAK,CAAC,GAGtEP,EAAkBO,CAAK,IACzBY,EAAuBA,EAAqB,MAAMnB,EAAkBO,CAAK,CAAC,EAE9E,CAEA,IAAIa,EAAoB,OAAO,KAAKxB,CAAc,EAC9CyB,EAAU,CAAC,EACXC,EAAU,OAAO,OAAO,IAAI,EAYhC,GAAI5B,EAAM,UAAU,EAAG,CACrB0B,EAAoB,OAAO,KAAK,KAAK,YAAY,EAEjD,QAAS3J,EAAI,EAAGA,EAAI2J,EAAkB,OAAQ3J,IAAK,CACjD,IAAIuJ,EAAmBI,EAAkB3J,CAAC,EACtCF,EAAWf,EAAK,SAAS,WAAWwK,CAAgB,EACxDpB,EAAeoB,CAAgB,EAAI,IAAIxK,EAAK,SAC9C,CACF,CAEA,QAASiB,EAAI,EAAGA,EAAI2J,EAAkB,OAAQ3J,IAAK,CASjD,IAAIF,EAAWf,EAAK,SAAS,WAAW4K,EAAkB3J,CAAC,CAAC,EACxDP,EAASK,EAAS,OAEtB,GAAK2J,EAAmB,SAAShK,CAAM,GAInC,CAAAiK,EAAqB,SAASjK,CAAM,EAIxC,KAAIqK,EAAc,KAAK,aAAahK,CAAQ,EACxCiK,EAAQ3B,EAAatI,EAAS,SAAS,EAAE,WAAWgK,CAAW,EAC/DE,EAEJ,IAAKA,EAAWH,EAAQpK,CAAM,KAAO,OACnCuK,EAAS,OAASD,EAClBC,EAAS,UAAU,QAAQ7B,EAAerI,CAAQ,CAAC,MAC9C,CACL,IAAImK,EAAQ,CACV,IAAKxK,EACL,MAAOsK,EACP,UAAW5B,EAAerI,CAAQ,CACpC,EACA+J,EAAQpK,CAAM,EAAIwK,EAClBL,EAAQ,KAAKK,CAAK,CACpB,EACF,CAKA,OAAOL,EAAQ,KAAK,SAAUzJ,GAAGC,GAAG,CAClC,OAAOA,GAAE,MAAQD,GAAE,KACrB,CAAC,CACH,EAUApB,EAAK,MAAM,UAAU,OAAS,UAAY,CACxC,IAAImL,EAAgB,OAAO,KAAK,KAAK,aAAa,EAC/C,KAAK,EACL,IAAI,SAAUvB,EAAM,CACnB,MAAO,CAACA,EAAM,KAAK,cAAcA,CAAI,CAAC,CACxC,EAAG,IAAI,EAELwB,EAAe,OAAO,KAAK,KAAK,YAAY,EAC7C,IAAI,SAAUC,EAAK,CAClB,MAAO,CAACA,EAAK,KAAK,aAAaA,CAAG,EAAE,OAAO,CAAC,CAC9C,EAAG,IAAI,EAET,MAAO,CACL,QAASrL,EAAK,QACd,OAAQ,KAAK,OACb,aAAcoL,EACd,cAAeD,EACf,SAAU,KAAK,SAAS,OAAO,CACjC,CACF,EAQAnL,EAAK,MAAM,KAAO,SAAUsL,EAAiB,CAC3C,IAAItC,EAAQ,CAAC,EACToC,EAAe,CAAC,EAChBG,EAAoBD,EAAgB,aACpCH,EAAgB,OAAO,OAAO,IAAI,EAClCK,EAA0BF,EAAgB,cAC1CG,EAAkB,IAAIzL,EAAK,SAAS,QACpC0C,EAAW1C,EAAK,SAAS,KAAKsL,EAAgB,QAAQ,EAEtDA,EAAgB,SAAWtL,EAAK,SAClCA,EAAK,MAAM,KAAK,4EAA8EA,EAAK,QAAU,sCAAwCsL,EAAgB,QAAU,GAAG,EAGpL,QAASrK,EAAI,EAAGA,EAAIsK,EAAkB,OAAQtK,IAAK,CACjD,IAAIyK,EAAQH,EAAkBtK,CAAC,EAC3BoK,EAAMK,EAAM,CAAC,EACb1K,EAAW0K,EAAM,CAAC,EAEtBN,EAAaC,CAAG,EAAI,IAAIrL,EAAK,OAAOgB,CAAQ,CAC9C,CAEA,QAASC,EAAI,EAAGA,EAAIuK,EAAwB,OAAQvK,IAAK,CACvD,IAAIyK,EAAQF,EAAwBvK,CAAC,EACjC2I,EAAO8B,EAAM,CAAC,EACdlK,EAAUkK,EAAM,CAAC,EAErBD,EAAgB,OAAO7B,CAAI,EAC3BuB,EAAcvB,CAAI,EAAIpI,CACxB,CAEA,OAAAiK,EAAgB,OAAO,EAEvBzC,EAAM,OAASsC,EAAgB,OAE/BtC,EAAM,aAAeoC,EACrBpC,EAAM,cAAgBmC,EACtBnC,EAAM,SAAWyC,EAAgB,KACjCzC,EAAM,SAAWtG,EAEV,IAAI1C,EAAK,MAAMgJ,CAAK,CAC7B,EACA;AAAA;AAAA;AAAA,GA6BAhJ,EAAK,QAAU,UAAY,CACzB,KAAK,KAAO,KACZ,KAAK,QAAU,OAAO,OAAO,IAAI,EACjC,KAAK,WAAa,OAAO,OAAO,IAAI,EACpC,KAAK,cAAgB,OAAO,OAAO,IAAI,EACvC,KAAK,qBAAuB,CAAC,EAC7B,KAAK,aAAe,CAAC,EACrB,KAAK,UAAYA,EAAK,UACtB,KAAK,SAAW,IAAIA,EAAK,SACzB,KAAK,eAAiB,IAAIA,EAAK,SAC/B,KAAK,cAAgB,EACrB,KAAK,GAAK,IACV,KAAK,IAAM,IACX,KAAK,UAAY,EACjB,KAAK,kBAAoB,CAAC,CAC5B,EAcAA,EAAK,QAAQ,UAAU,IAAM,SAAUqL,EAAK,CAC1C,KAAK,KAAOA,CACd,EAkCArL,EAAK,QAAQ,UAAU,MAAQ,SAAUW,EAAWgL,EAAY,CAC9D,GAAI,KAAK,KAAKhL,CAAS,EACrB,MAAM,IAAI,WAAY,UAAYA,EAAY,kCAAkC,EAGlF,KAAK,QAAQA,CAAS,EAAIgL,GAAc,CAAC,CAC3C,EAUA3L,EAAK,QAAQ,UAAU,EAAI,SAAU4L,EAAQ,CACvCA,EAAS,EACX,KAAK,GAAK,EACDA,EAAS,EAClB,KAAK,GAAK,EAEV,KAAK,GAAKA,CAEd,EASA5L,EAAK,QAAQ,UAAU,GAAK,SAAU4L,EAAQ,CAC5C,KAAK,IAAMA,CACb,EAmBA5L,EAAK,QAAQ,UAAU,IAAM,SAAU6L,EAAKF,EAAY,CACtD,IAAIjL,EAASmL,EAAI,KAAK,IAAI,EACtBC,EAAS,OAAO,KAAK,KAAK,OAAO,EAErC,KAAK,WAAWpL,CAAM,EAAIiL,GAAc,CAAC,EACzC,KAAK,eAAiB,EAEtB,QAAS1K,EAAI,EAAGA,EAAI6K,EAAO,OAAQ7K,IAAK,CACtC,IAAIN,EAAYmL,EAAO7K,CAAC,EACpB8K,EAAY,KAAK,QAAQpL,CAAS,EAAE,UACpCoJ,EAAQgC,EAAYA,EAAUF,CAAG,EAAIA,EAAIlL,CAAS,EAClDsB,EAAS,KAAK,UAAU8H,EAAO,CAC7B,OAAQ,CAACpJ,CAAS,CACpB,CAAC,EACD8I,EAAQ,KAAK,SAAS,IAAIxH,CAAM,EAChClB,EAAW,IAAIf,EAAK,SAAUU,EAAQC,CAAS,EAC/CqL,EAAa,OAAO,OAAO,IAAI,EAEnC,KAAK,qBAAqBjL,CAAQ,EAAIiL,EACtC,KAAK,aAAajL,CAAQ,EAAI,EAG9B,KAAK,aAAaA,CAAQ,GAAK0I,EAAM,OAGrC,QAASvG,EAAI,EAAGA,EAAIuG,EAAM,OAAQvG,IAAK,CACrC,IAAI0G,EAAOH,EAAMvG,CAAC,EAUlB,GARI8I,EAAWpC,CAAI,GAAK,OACtBoC,EAAWpC,CAAI,EAAI,GAGrBoC,EAAWpC,CAAI,GAAK,EAIhB,KAAK,cAAcA,CAAI,GAAK,KAAW,CACzC,IAAIpI,EAAU,OAAO,OAAO,IAAI,EAChCA,EAAQ,OAAY,KAAK,UACzB,KAAK,WAAa,EAElB,QAAS4B,EAAI,EAAGA,EAAI0I,EAAO,OAAQ1I,IACjC5B,EAAQsK,EAAO1I,CAAC,CAAC,EAAI,OAAO,OAAO,IAAI,EAGzC,KAAK,cAAcwG,CAAI,EAAIpI,CAC7B,CAGI,KAAK,cAAcoI,CAAI,EAAEjJ,CAAS,EAAED,CAAM,GAAK,OACjD,KAAK,cAAckJ,CAAI,EAAEjJ,CAAS,EAAED,CAAM,EAAI,OAAO,OAAO,IAAI,GAKlE,QAAS4J,EAAI,EAAGA,EAAI,KAAK,kBAAkB,OAAQA,IAAK,CACtD,IAAI2B,EAAc,KAAK,kBAAkB3B,CAAC,EACtCzI,EAAW+H,EAAK,SAASqC,CAAW,EAEpC,KAAK,cAAcrC,CAAI,EAAEjJ,CAAS,EAAED,CAAM,EAAEuL,CAAW,GAAK,OAC9D,KAAK,cAAcrC,CAAI,EAAEjJ,CAAS,EAAED,CAAM,EAAEuL,CAAW,EAAI,CAAC,GAG9D,KAAK,cAAcrC,CAAI,EAAEjJ,CAAS,EAAED,CAAM,EAAEuL,CAAW,EAAE,KAAKpK,CAAQ,CACxE,CACF,CAEF,CACF,EAOA7B,EAAK,QAAQ,UAAU,6BAA+B,UAAY,CAOhE,QALIkM,EAAY,OAAO,KAAK,KAAK,YAAY,EACzCC,EAAiBD,EAAU,OAC3BE,EAAc,CAAC,EACfC,EAAqB,CAAC,EAEjBpL,EAAI,EAAGA,EAAIkL,EAAgBlL,IAAK,CACvC,IAAIF,EAAWf,EAAK,SAAS,WAAWkM,EAAUjL,CAAC,CAAC,EAChD8I,EAAQhJ,EAAS,UAErBsL,EAAmBtC,CAAK,IAAMsC,EAAmBtC,CAAK,EAAI,GAC1DsC,EAAmBtC,CAAK,GAAK,EAE7BqC,EAAYrC,CAAK,IAAMqC,EAAYrC,CAAK,EAAI,GAC5CqC,EAAYrC,CAAK,GAAK,KAAK,aAAahJ,CAAQ,CAClD,CAIA,QAFI+K,EAAS,OAAO,KAAK,KAAK,OAAO,EAE5B7K,EAAI,EAAGA,EAAI6K,EAAO,OAAQ7K,IAAK,CACtC,IAAIN,EAAYmL,EAAO7K,CAAC,EACxBmL,EAAYzL,CAAS,EAAIyL,EAAYzL,CAAS,EAAI0L,EAAmB1L,CAAS,CAChF,CAEA,KAAK,mBAAqByL,CAC5B,EAOApM,EAAK,QAAQ,UAAU,mBAAqB,UAAY,CAMtD,QALIoL,EAAe,CAAC,EAChBc,EAAY,OAAO,KAAK,KAAK,oBAAoB,EACjDI,EAAkBJ,EAAU,OAC5BK,EAAe,OAAO,OAAO,IAAI,EAE5BtL,EAAI,EAAGA,EAAIqL,EAAiBrL,IAAK,CAaxC,QAZIF,EAAWf,EAAK,SAAS,WAAWkM,EAAUjL,CAAC,CAAC,EAChDN,EAAYI,EAAS,UACrByL,EAAc,KAAK,aAAazL,CAAQ,EACxCgK,EAAc,IAAI/K,EAAK,OACvByM,EAAkB,KAAK,qBAAqB1L,CAAQ,EACpD0I,EAAQ,OAAO,KAAKgD,CAAe,EACnCC,EAAcjD,EAAM,OAGpBkD,EAAa,KAAK,QAAQhM,CAAS,EAAE,OAAS,EAC9CiM,EAAW,KAAK,WAAW7L,EAAS,MAAM,EAAE,OAAS,EAEhDmC,EAAI,EAAGA,EAAIwJ,EAAaxJ,IAAK,CACpC,IAAI0G,EAAOH,EAAMvG,CAAC,EACd2J,EAAKJ,EAAgB7C,CAAI,EACzBK,EAAY,KAAK,cAAcL,CAAI,EAAE,OACrCkD,EAAK9B,EAAO+B,EAEZR,EAAa3C,CAAI,IAAM,QACzBkD,EAAM9M,EAAK,IAAI,KAAK,cAAc4J,CAAI,EAAG,KAAK,aAAa,EAC3D2C,EAAa3C,CAAI,EAAIkD,GAErBA,EAAMP,EAAa3C,CAAI,EAGzBoB,EAAQ8B,IAAQ,KAAK,IAAM,GAAKD,IAAO,KAAK,KAAO,EAAI,KAAK,GAAK,KAAK,IAAML,EAAc,KAAK,mBAAmB7L,CAAS,IAAMkM,GACjI7B,GAAS2B,EACT3B,GAAS4B,EACTG,EAAqB,KAAK,MAAM/B,EAAQ,GAAI,EAAI,IAQhDD,EAAY,OAAOd,EAAW8C,CAAkB,CAClD,CAEA3B,EAAarK,CAAQ,EAAIgK,CAC3B,CAEA,KAAK,aAAeK,CACtB,EAOApL,EAAK,QAAQ,UAAU,eAAiB,UAAY,CAClD,KAAK,SAAWA,EAAK,SAAS,UAC5B,OAAO,KAAK,KAAK,aAAa,EAAE,KAAK,CACvC,CACF,EAUAA,EAAK,QAAQ,UAAU,MAAQ,UAAY,CACzC,YAAK,6BAA6B,EAClC,KAAK,mBAAmB,EACxB,KAAK,eAAe,EAEb,IAAIA,EAAK,MAAM,CACpB,cAAe,KAAK,cACpB,aAAc,KAAK,aACnB,SAAU,KAAK,SACf,OAAQ,OAAO,KAAK,KAAK,OAAO,EAChC,SAAU,KAAK,cACjB,CAAC,CACH,EAgBAA,EAAK,QAAQ,UAAU,IAAM,SAAU8B,EAAI,CACzC,IAAIkL,EAAO,MAAM,UAAU,MAAM,KAAK,UAAW,CAAC,EAClDA,EAAK,QAAQ,IAAI,EACjBlL,EAAG,MAAM,KAAMkL,CAAI,CACrB,EAaAhN,EAAK,UAAY,SAAU4J,EAAMG,EAAOlI,EAAU,CAShD,QARIoL,EAAiB,OAAO,OAAO,IAAI,EACnCC,EAAe,OAAO,KAAKrL,GAAY,CAAC,CAAC,EAOpCZ,EAAI,EAAGA,EAAIiM,EAAa,OAAQjM,IAAK,CAC5C,IAAIT,EAAM0M,EAAajM,CAAC,EACxBgM,EAAezM,CAAG,EAAIqB,EAASrB,CAAG,EAAE,MAAM,CAC5C,CAEA,KAAK,SAAW,OAAO,OAAO,IAAI,EAE9BoJ,IAAS,SACX,KAAK,SAASA,CAAI,EAAI,OAAO,OAAO,IAAI,EACxC,KAAK,SAASA,CAAI,EAAEG,CAAK,EAAIkD,EAEjC,EAWAjN,EAAK,UAAU,UAAU,QAAU,SAAUmN,EAAgB,CAG3D,QAFI1D,EAAQ,OAAO,KAAK0D,EAAe,QAAQ,EAEtClM,EAAI,EAAGA,EAAIwI,EAAM,OAAQxI,IAAK,CACrC,IAAI2I,EAAOH,EAAMxI,CAAC,EACd6K,EAAS,OAAO,KAAKqB,EAAe,SAASvD,CAAI,CAAC,EAElD,KAAK,SAASA,CAAI,GAAK,OACzB,KAAK,SAASA,CAAI,EAAI,OAAO,OAAO,IAAI,GAG1C,QAAS1G,EAAI,EAAGA,EAAI4I,EAAO,OAAQ5I,IAAK,CACtC,IAAI6G,EAAQ+B,EAAO5I,CAAC,EAChB3C,EAAO,OAAO,KAAK4M,EAAe,SAASvD,CAAI,EAAEG,CAAK,CAAC,EAEvD,KAAK,SAASH,CAAI,EAAEG,CAAK,GAAK,OAChC,KAAK,SAASH,CAAI,EAAEG,CAAK,EAAI,OAAO,OAAO,IAAI,GAGjD,QAAS3G,EAAI,EAAGA,EAAI7C,EAAK,OAAQ6C,IAAK,CACpC,IAAI5C,EAAMD,EAAK6C,CAAC,EAEZ,KAAK,SAASwG,CAAI,EAAEG,CAAK,EAAEvJ,CAAG,GAAK,KACrC,KAAK,SAASoJ,CAAI,EAAEG,CAAK,EAAEvJ,CAAG,EAAI2M,EAAe,SAASvD,CAAI,EAAEG,CAAK,EAAEvJ,CAAG,EAE1E,KAAK,SAASoJ,CAAI,EAAEG,CAAK,EAAEvJ,CAAG,EAAI,KAAK,SAASoJ,CAAI,EAAEG,CAAK,EAAEvJ,CAAG,EAAE,OAAO2M,EAAe,SAASvD,CAAI,EAAEG,CAAK,EAAEvJ,CAAG,CAAC,CAGtH,CACF,CACF,CACF,EASAR,EAAK,UAAU,UAAU,IAAM,SAAU4J,EAAMG,EAAOlI,EAAU,CAC9D,GAAI,EAAE+H,KAAQ,KAAK,UAAW,CAC5B,KAAK,SAASA,CAAI,EAAI,OAAO,OAAO,IAAI,EACxC,KAAK,SAASA,CAAI,EAAEG,CAAK,EAAIlI,EAC7B,MACF,CAEA,GAAI,EAAEkI,KAAS,KAAK,SAASH,CAAI,GAAI,CACnC,KAAK,SAASA,CAAI,EAAEG,CAAK,EAAIlI,EAC7B,MACF,CAIA,QAFIqL,EAAe,OAAO,KAAKrL,CAAQ,EAE9BZ,EAAI,EAAGA,EAAIiM,EAAa,OAAQjM,IAAK,CAC5C,IAAIT,EAAM0M,EAAajM,CAAC,EAEpBT,KAAO,KAAK,SAASoJ,CAAI,EAAEG,CAAK,EAClC,KAAK,SAASH,CAAI,EAAEG,CAAK,EAAEvJ,CAAG,EAAI,KAAK,SAASoJ,CAAI,EAAEG,CAAK,EAAEvJ,CAAG,EAAE,OAAOqB,EAASrB,CAAG,CAAC,EAEtF,KAAK,SAASoJ,CAAI,EAAEG,CAAK,EAAEvJ,CAAG,EAAIqB,EAASrB,CAAG,CAElD,CACF,EAYAR,EAAK,MAAQ,SAAUoN,EAAW,CAChC,KAAK,QAAU,CAAC,EAChB,KAAK,UAAYA,CACnB,EA0BApN,EAAK,MAAM,SAAW,IAAI,OAAQ,GAAG,EACrCA,EAAK,MAAM,SAAS,KAAO,EAC3BA,EAAK,MAAM,SAAS,QAAU,EAC9BA,EAAK,MAAM,SAAS,SAAW,EAa/BA,EAAK,MAAM,SAAW,CAIpB,SAAU,EAMV,SAAU,EAMV,WAAY,CACd,EAyBAA,EAAK,MAAM,UAAU,OAAS,SAAUkH,EAAQ,CAC9C,MAAM,WAAYA,IAChBA,EAAO,OAAS,KAAK,WAGjB,UAAWA,IACfA,EAAO,MAAQ,GAGX,gBAAiBA,IACrBA,EAAO,YAAc,IAGjB,aAAcA,IAClBA,EAAO,SAAWlH,EAAK,MAAM,SAAS,MAGnCkH,EAAO,SAAWlH,EAAK,MAAM,SAAS,SAAakH,EAAO,KAAK,OAAO,CAAC,GAAKlH,EAAK,MAAM,WAC1FkH,EAAO,KAAO,IAAMA,EAAO,MAGxBA,EAAO,SAAWlH,EAAK,MAAM,SAAS,UAAckH,EAAO,KAAK,MAAM,EAAE,GAAKlH,EAAK,MAAM,WAC3FkH,EAAO,KAAO,GAAKA,EAAO,KAAO,KAG7B,aAAcA,IAClBA,EAAO,SAAWlH,EAAK,MAAM,SAAS,UAGxC,KAAK,QAAQ,KAAKkH,CAAM,EAEjB,IACT,EASAlH,EAAK,MAAM,UAAU,UAAY,UAAY,CAC3C,QAASiB,EAAI,EAAGA,EAAI,KAAK,QAAQ,OAAQA,IACvC,GAAI,KAAK,QAAQA,CAAC,EAAE,UAAYjB,EAAK,MAAM,SAAS,WAClD,MAAO,GAIX,MAAO,EACT,EA4BAA,EAAK,MAAM,UAAU,KAAO,SAAU4J,EAAMyD,EAAS,CACnD,GAAI,MAAM,QAAQzD,CAAI,EACpB,OAAAA,EAAK,QAAQ,SAAU7H,EAAG,CAAE,KAAK,KAAKA,EAAG/B,EAAK,MAAM,MAAMqN,CAAO,CAAC,CAAE,EAAG,IAAI,EACpE,KAGT,IAAInG,EAASmG,GAAW,CAAC,EACzB,OAAAnG,EAAO,KAAO0C,EAAK,SAAS,EAE5B,KAAK,OAAO1C,CAAM,EAEX,IACT,EACAlH,EAAK,gBAAkB,SAAUI,EAASmD,EAAOC,EAAK,CACpD,KAAK,KAAO,kBACZ,KAAK,QAAUpD,EACf,KAAK,MAAQmD,EACb,KAAK,IAAMC,CACb,EAEAxD,EAAK,gBAAgB,UAAY,IAAI,MACrCA,EAAK,WAAa,SAAU4B,EAAK,CAC/B,KAAK,QAAU,CAAC,EAChB,KAAK,IAAMA,EACX,KAAK,OAASA,EAAI,OAClB,KAAK,IAAM,EACX,KAAK,MAAQ,EACb,KAAK,oBAAsB,CAAC,CAC9B,EAEA5B,EAAK,WAAW,UAAU,IAAM,UAAY,CAG1C,QAFIsN,EAAQtN,EAAK,WAAW,QAErBsN,GACLA,EAAQA,EAAM,IAAI,CAEtB,EAEAtN,EAAK,WAAW,UAAU,YAAc,UAAY,CAKlD,QAJIuN,EAAY,CAAC,EACbpL,EAAa,KAAK,MAClBD,EAAW,KAAK,IAEX,EAAI,EAAG,EAAI,KAAK,oBAAoB,OAAQ,IACnDA,EAAW,KAAK,oBAAoB,CAAC,EACrCqL,EAAU,KAAK,KAAK,IAAI,MAAMpL,EAAYD,CAAQ,CAAC,EACnDC,EAAaD,EAAW,EAG1B,OAAAqL,EAAU,KAAK,KAAK,IAAI,MAAMpL,EAAY,KAAK,GAAG,CAAC,EACnD,KAAK,oBAAoB,OAAS,EAE3BoL,EAAU,KAAK,EAAE,CAC1B,EAEAvN,EAAK,WAAW,UAAU,KAAO,SAAUwN,EAAM,CAC/C,KAAK,QAAQ,KAAK,CAChB,KAAMA,EACN,IAAK,KAAK,YAAY,EACtB,MAAO,KAAK,MACZ,IAAK,KAAK,GACZ,CAAC,EAED,KAAK,MAAQ,KAAK,GACpB,EAEAxN,EAAK,WAAW,UAAU,gBAAkB,UAAY,CACtD,KAAK,oBAAoB,KAAK,KAAK,IAAM,CAAC,EAC1C,KAAK,KAAO,CACd,EAEAA,EAAK,WAAW,UAAU,KAAO,UAAY,CAC3C,GAAI,KAAK,KAAO,KAAK,OACnB,OAAOA,EAAK,WAAW,IAGzB,IAAIoC,EAAO,KAAK,IAAI,OAAO,KAAK,GAAG,EACnC,YAAK,KAAO,EACLA,CACT,EAEApC,EAAK,WAAW,UAAU,MAAQ,UAAY,CAC5C,OAAO,KAAK,IAAM,KAAK,KACzB,EAEAA,EAAK,WAAW,UAAU,OAAS,UAAY,CACzC,KAAK,OAAS,KAAK,MACrB,KAAK,KAAO,GAGd,KAAK,MAAQ,KAAK,GACpB,EAEAA,EAAK,WAAW,UAAU,OAAS,UAAY,CAC7C,KAAK,KAAO,CACd,EAEAA,EAAK,WAAW,UAAU,eAAiB,UAAY,CACrD,IAAIoC,EAAMqL,EAEV,GACErL,EAAO,KAAK,KAAK,EACjBqL,EAAWrL,EAAK,WAAW,CAAC,QACrBqL,EAAW,IAAMA,EAAW,IAEjCrL,GAAQpC,EAAK,WAAW,KAC1B,KAAK,OAAO,CAEhB,EAEAA,EAAK,WAAW,UAAU,KAAO,UAAY,CAC3C,OAAO,KAAK,IAAM,KAAK,MACzB,EAEAA,EAAK,WAAW,IAAM,MACtBA,EAAK,WAAW,MAAQ,QACxBA,EAAK,WAAW,KAAO,OACvBA,EAAK,WAAW,cAAgB,gBAChCA,EAAK,WAAW,MAAQ,QACxBA,EAAK,WAAW,SAAW,WAE3BA,EAAK,WAAW,SAAW,SAAU0N,EAAO,CAC1C,OAAAA,EAAM,OAAO,EACbA,EAAM,KAAK1N,EAAK,WAAW,KAAK,EAChC0N,EAAM,OAAO,EACN1N,EAAK,WAAW,OACzB,EAEAA,EAAK,WAAW,QAAU,SAAU0N,EAAO,CAQzC,GAPIA,EAAM,MAAM,EAAI,IAClBA,EAAM,OAAO,EACbA,EAAM,KAAK1N,EAAK,WAAW,IAAI,GAGjC0N,EAAM,OAAO,EAETA,EAAM,KAAK,EACb,OAAO1N,EAAK,WAAW,OAE3B,EAEAA,EAAK,WAAW,gBAAkB,SAAU0N,EAAO,CACjD,OAAAA,EAAM,OAAO,EACbA,EAAM,eAAe,EACrBA,EAAM,KAAK1N,EAAK,WAAW,aAAa,EACjCA,EAAK,WAAW,OACzB,EAEAA,EAAK,WAAW,SAAW,SAAU0N,EAAO,CAC1C,OAAAA,EAAM,OAAO,EACbA,EAAM,eAAe,EACrBA,EAAM,KAAK1N,EAAK,WAAW,KAAK,EACzBA,EAAK,WAAW,OACzB,EAEAA,EAAK,WAAW,OAAS,SAAU0N,EAAO,CACpCA,EAAM,MAAM,EAAI,GAClBA,EAAM,KAAK1N,EAAK,WAAW,IAAI,CAEnC,EAaAA,EAAK,WAAW,cAAgBA,EAAK,UAAU,UAE/CA,EAAK,WAAW,QAAU,SAAU0N,EAAO,CACzC,OAAa,CACX,IAAItL,EAAOsL,EAAM,KAAK,EAEtB,GAAItL,GAAQpC,EAAK,WAAW,IAC1B,OAAOA,EAAK,WAAW,OAIzB,GAAIoC,EAAK,WAAW,CAAC,GAAK,GAAI,CAC5BsL,EAAM,gBAAgB,EACtB,QACF,CAEA,GAAItL,GAAQ,IACV,OAAOpC,EAAK,WAAW,SAGzB,GAAIoC,GAAQ,IACV,OAAAsL,EAAM,OAAO,EACTA,EAAM,MAAM,EAAI,GAClBA,EAAM,KAAK1N,EAAK,WAAW,IAAI,EAE1BA,EAAK,WAAW,gBAGzB,GAAIoC,GAAQ,IACV,OAAAsL,EAAM,OAAO,EACTA,EAAM,MAAM,EAAI,GAClBA,EAAM,KAAK1N,EAAK,WAAW,IAAI,EAE1BA,EAAK,WAAW,SAczB,GARIoC,GAAQ,KAAOsL,EAAM,MAAM,IAAM,GAQjCtL,GAAQ,KAAOsL,EAAM,MAAM,IAAM,EACnC,OAAAA,EAAM,KAAK1N,EAAK,WAAW,QAAQ,EAC5BA,EAAK,WAAW,QAGzB,GAAIoC,EAAK,MAAMpC,EAAK,WAAW,aAAa,EAC1C,OAAOA,EAAK,WAAW,OAE3B,CACF,EAEAA,EAAK,YAAc,SAAU4B,EAAKsH,EAAO,CACvC,KAAK,MAAQ,IAAIlJ,EAAK,WAAY4B,CAAG,EACrC,KAAK,MAAQsH,EACb,KAAK,cAAgB,CAAC,EACtB,KAAK,UAAY,CACnB,EAEAlJ,EAAK,YAAY,UAAU,MAAQ,UAAY,CAC7C,KAAK,MAAM,IAAI,EACf,KAAK,QAAU,KAAK,MAAM,QAI1B,QAFIsN,EAAQtN,EAAK,YAAY,YAEtBsN,GACLA,EAAQA,EAAM,IAAI,EAGpB,OAAO,KAAK,KACd,EAEAtN,EAAK,YAAY,UAAU,WAAa,UAAY,CAClD,OAAO,KAAK,QAAQ,KAAK,SAAS,CACpC,EAEAA,EAAK,YAAY,UAAU,cAAgB,UAAY,CACrD,IAAI2N,EAAS,KAAK,WAAW,EAC7B,YAAK,WAAa,EACXA,CACT,EAEA3N,EAAK,YAAY,UAAU,WAAa,UAAY,CAClD,IAAI4N,EAAkB,KAAK,cAC3B,KAAK,MAAM,OAAOA,CAAe,EACjC,KAAK,cAAgB,CAAC,CACxB,EAEA5N,EAAK,YAAY,YAAc,SAAUmJ,EAAQ,CAC/C,IAAIwE,EAASxE,EAAO,WAAW,EAE/B,GAAIwE,GAAU,KAId,OAAQA,EAAO,KAAM,CACnB,KAAK3N,EAAK,WAAW,SACnB,OAAOA,EAAK,YAAY,cAC1B,KAAKA,EAAK,WAAW,MACnB,OAAOA,EAAK,YAAY,WAC1B,KAAKA,EAAK,WAAW,KACnB,OAAOA,EAAK,YAAY,UAC1B,QACE,IAAI6N,EAAe,4CAA8CF,EAAO,KAExE,MAAIA,EAAO,IAAI,QAAU,IACvBE,GAAgB,gBAAkBF,EAAO,IAAM,KAG3C,IAAI3N,EAAK,gBAAiB6N,EAAcF,EAAO,MAAOA,EAAO,GAAG,CAC1E,CACF,EAEA3N,EAAK,YAAY,cAAgB,SAAUmJ,EAAQ,CACjD,IAAIwE,EAASxE,EAAO,cAAc,EAElC,GAAIwE,GAAU,KAId,QAAQA,EAAO,IAAK,CAClB,IAAK,IACHxE,EAAO,cAAc,SAAWnJ,EAAK,MAAM,SAAS,WACpD,MACF,IAAK,IACHmJ,EAAO,cAAc,SAAWnJ,EAAK,MAAM,SAAS,SACpD,MACF,QACE,IAAI6N,EAAe,kCAAoCF,EAAO,IAAM,IACpE,MAAM,IAAI3N,EAAK,gBAAiB6N,EAAcF,EAAO,MAAOA,EAAO,GAAG,CAC1E,CAEA,IAAIG,EAAa3E,EAAO,WAAW,EAEnC,GAAI2E,GAAc,KAAW,CAC3B,IAAID,EAAe,yCACnB,MAAM,IAAI7N,EAAK,gBAAiB6N,EAAcF,EAAO,MAAOA,EAAO,GAAG,CACxE,CAEA,OAAQG,EAAW,KAAM,CACvB,KAAK9N,EAAK,WAAW,MACnB,OAAOA,EAAK,YAAY,WAC1B,KAAKA,EAAK,WAAW,KACnB,OAAOA,EAAK,YAAY,UAC1B,QACE,IAAI6N,EAAe,mCAAqCC,EAAW,KAAO,IAC1E,MAAM,IAAI9N,EAAK,gBAAiB6N,EAAcC,EAAW,MAAOA,EAAW,GAAG,CAClF,EACF,EAEA9N,EAAK,YAAY,WAAa,SAAUmJ,EAAQ,CAC9C,IAAIwE,EAASxE,EAAO,cAAc,EAElC,GAAIwE,GAAU,KAId,IAAIxE,EAAO,MAAM,UAAU,QAAQwE,EAAO,GAAG,GAAK,GAAI,CACpD,IAAII,EAAiB5E,EAAO,MAAM,UAAU,IAAI,SAAU6E,EAAG,CAAE,MAAO,IAAMA,EAAI,GAAI,CAAC,EAAE,KAAK,IAAI,EAC5FH,EAAe,uBAAyBF,EAAO,IAAM,uBAAyBI,EAElF,MAAM,IAAI/N,EAAK,gBAAiB6N,EAAcF,EAAO,MAAOA,EAAO,GAAG,CACxE,CAEAxE,EAAO,cAAc,OAAS,CAACwE,EAAO,GAAG,EAEzC,IAAIG,EAAa3E,EAAO,WAAW,EAEnC,GAAI2E,GAAc,KAAW,CAC3B,IAAID,EAAe,gCACnB,MAAM,IAAI7N,EAAK,gBAAiB6N,EAAcF,EAAO,MAAOA,EAAO,GAAG,CACxE,CAEA,OAAQG,EAAW,KAAM,CACvB,KAAK9N,EAAK,WAAW,KACnB,OAAOA,EAAK,YAAY,UAC1B,QACE,IAAI6N,EAAe,0BAA4BC,EAAW,KAAO,IACjE,MAAM,IAAI9N,EAAK,gBAAiB6N,EAAcC,EAAW,MAAOA,EAAW,GAAG,CAClF,EACF,EAEA9N,EAAK,YAAY,UAAY,SAAUmJ,EAAQ,CAC7C,IAAIwE,EAASxE,EAAO,cAAc,EAElC,GAAIwE,GAAU,KAId,CAAAxE,EAAO,cAAc,KAAOwE,EAAO,IAAI,YAAY,EAE/CA,EAAO,IAAI,QAAQ,GAAG,GAAK,KAC7BxE,EAAO,cAAc,YAAc,IAGrC,IAAI2E,EAAa3E,EAAO,WAAW,EAEnC,GAAI2E,GAAc,KAAW,CAC3B3E,EAAO,WAAW,EAClB,MACF,CAEA,OAAQ2E,EAAW,KAAM,CACvB,KAAK9N,EAAK,WAAW,KACnB,OAAAmJ,EAAO,WAAW,EACXnJ,EAAK,YAAY,UAC1B,KAAKA,EAAK,WAAW,MACnB,OAAAmJ,EAAO,WAAW,EACXnJ,EAAK,YAAY,WAC1B,KAAKA,EAAK,WAAW,cACnB,OAAOA,EAAK,YAAY,kBAC1B,KAAKA,EAAK,WAAW,MACnB,OAAOA,EAAK,YAAY,WAC1B,KAAKA,EAAK,WAAW,SACnB,OAAAmJ,EAAO,WAAW,EACXnJ,EAAK,YAAY,cAC1B,QACE,IAAI6N,EAAe,2BAA6BC,EAAW,KAAO,IAClE,MAAM,IAAI9N,EAAK,gBAAiB6N,EAAcC,EAAW,MAAOA,EAAW,GAAG,CAClF,EACF,EAEA9N,EAAK,YAAY,kBAAoB,SAAUmJ,EAAQ,CACrD,IAAIwE,EAASxE,EAAO,cAAc,EAElC,GAAIwE,GAAU,KAId,KAAIxG,EAAe,SAASwG,EAAO,IAAK,EAAE,EAE1C,GAAI,MAAMxG,CAAY,EAAG,CACvB,IAAI0G,EAAe,gCACnB,MAAM,IAAI7N,EAAK,gBAAiB6N,EAAcF,EAAO,MAAOA,EAAO,GAAG,CACxE,CAEAxE,EAAO,cAAc,aAAehC,EAEpC,IAAI2G,EAAa3E,EAAO,WAAW,EAEnC,GAAI2E,GAAc,KAAW,CAC3B3E,EAAO,WAAW,EAClB,MACF,CAEA,OAAQ2E,EAAW,KAAM,CACvB,KAAK9N,EAAK,WAAW,KACnB,OAAAmJ,EAAO,WAAW,EACXnJ,EAAK,YAAY,UAC1B,KAAKA,EAAK,WAAW,MACnB,OAAAmJ,EAAO,WAAW,EACXnJ,EAAK,YAAY,WAC1B,KAAKA,EAAK,WAAW,cACnB,OAAOA,EAAK,YAAY,kBAC1B,KAAKA,EAAK,WAAW,MACnB,OAAOA,EAAK,YAAY,WAC1B,KAAKA,EAAK,WAAW,SACnB,OAAAmJ,EAAO,WAAW,EACXnJ,EAAK,YAAY,cAC1B,QACE,IAAI6N,EAAe,2BAA6BC,EAAW,KAAO,IAClE,MAAM,IAAI9N,EAAK,gBAAiB6N,EAAcC,EAAW,MAAOA,EAAW,GAAG,CAClF,EACF,EAEA9N,EAAK,YAAY,WAAa,SAAUmJ,EAAQ,CAC9C,IAAIwE,EAASxE,EAAO,cAAc,EAElC,GAAIwE,GAAU,KAId,KAAIM,EAAQ,SAASN,EAAO,IAAK,EAAE,EAEnC,GAAI,MAAMM,CAAK,EAAG,CAChB,IAAIJ,EAAe,wBACnB,MAAM,IAAI7N,EAAK,gBAAiB6N,EAAcF,EAAO,MAAOA,EAAO,GAAG,CACxE,CAEAxE,EAAO,cAAc,MAAQ8E,EAE7B,IAAIH,EAAa3E,EAAO,WAAW,EAEnC,GAAI2E,GAAc,KAAW,CAC3B3E,EAAO,WAAW,EAClB,MACF,CAEA,OAAQ2E,EAAW,KAAM,CACvB,KAAK9N,EAAK,WAAW,KACnB,OAAAmJ,EAAO,WAAW,EACXnJ,EAAK,YAAY,UAC1B,KAAKA,EAAK,WAAW,MACnB,OAAAmJ,EAAO,WAAW,EACXnJ,EAAK,YAAY,WAC1B,KAAKA,EAAK,WAAW,cACnB,OAAOA,EAAK,YAAY,kBAC1B,KAAKA,EAAK,WAAW,MACnB,OAAOA,EAAK,YAAY,WAC1B,KAAKA,EAAK,WAAW,SACnB,OAAAmJ,EAAO,WAAW,EACXnJ,EAAK,YAAY,cAC1B,QACE,IAAI6N,EAAe,2BAA6BC,EAAW,KAAO,IAClE,MAAM,IAAI9N,EAAK,gBAAiB6N,EAAcC,EAAW,MAAOA,EAAW,GAAG,CAClF,EACF,EAMI,SAAU1G,EAAM8G,EAAS,CACrB,OAAO,QAAW,YAAc,OAAO,IAEzC,OAAOA,CAAO,EACL,OAAOpO,GAAY,SAM5BC,GAAO,QAAUmO,EAAQ,EAGzB9G,EAAK,KAAO8G,EAAQ,CAExB,EAAE,KAAM,UAAY,CAMlB,OAAOlO,CACT,CAAC,CACH,GAAG,IC53GH,IAAAmO,GAAiB,SCiDV,SAASC,GACdC,EAAkBC,EAAmB,SAClC,CACH,IAAMC,EAAKC,GAAsBH,EAAUC,CAAI,EAC/C,GAAI,OAAOC,GAAO,YAChB,MAAM,IAAI,eACR,8BAA8BF,CAAQ,iBACxC,EAGF,OAAOE,CACT,CAsBO,SAASC,GACdH,EAAkBC,EAAmB,SACtB,CACf,OAAOA,EAAK,cAAiBD,CAAQ,GAAK,MAC5C,CCjFK,OAAO,UACV,OAAO,QAAU,SAAUI,EAAa,CACtC,IAAMC,EAA2B,CAAC,EAClC,QAAWC,KAAO,OAAO,KAAKF,CAAG,EAE/BC,EAAK,KAAK,CAACC,EAAKF,EAAIE,CAAG,CAAC,CAAC,EAG3B,OAAOD,CACT,GAGG,OAAO,SACV,OAAO,OAAS,SAAUD,EAAa,CACrC,IAAMC,EAAiB,CAAC,EACxB,QAAWC,KAAO,OAAO,KAAKF,CAAG,EAE/BC,EAAK,KAAKD,EAAIE,CAAG,CAAC,EAGpB,OAAOD,CACT,GAKE,OAAO,SAAY,cAGhB,QAAQ,UAAU,WACrB,QAAQ,UAAU,SAAW,SAC3BE,EAA8BC,EACxB,CACF,OAAOD,GAAM,UACf,KAAK,WAAaA,EAAE,KACpB,KAAK,UAAYA,EAAE,MAEnB,KAAK,WAAaA,EAClB,KAAK,UAAYC,EAErB,GAGG,QAAQ,UAAU,cACrB,QAAQ,UAAU,YAAc,YAC3BC,EACG,CACN,IAAMC,EAAS,KAAK,WACpB,GAAIA,EAAQ,CACND,EAAM,SAAW,GACnBC,EAAO,YAAY,IAAI,EAGzB,QAASC,EAAIF,EAAM,OAAS,EAAGE,GAAK,EAAGA,IAAK,CAC1C,IAAIC,EAAOH,EAAME,CAAC,EACd,OAAOC,GAAS,SAClBA,EAAO,SAAS,eAAeA,CAAI,EAC5BA,EAAK,YACZA,EAAK,WAAW,YAAYA,CAAI,EAG7BD,EAGHD,EAAO,aAAa,KAAK,gBAAkBE,CAAI,EAF/CF,EAAO,aAAaE,EAAM,IAAI,CAGlC,CACF,CACF,ICDG,SAASC,GACdC,EAC6B,CAC7B,IAAMC,EAAM,IAAI,IAChB,QAAWC,KAAOF,EAAM,CACtB,GAAM,CAACG,CAAI,EAAID,EAAI,SAAS,MAAM,GAAG,EAG/BE,EAAUH,EAAI,IAAIE,CAAI,EACxB,OAAOC,GAAY,YACrBH,EAAI,IAAIE,EAAMD,CAAG,GAIjBD,EAAI,IAAIC,EAAI,SAAUA,CAAG,EACzBA,EAAI,OAASE,EAEjB,CAGA,OAAOH,CACT,CCnEO,SAASI,EACdC,EAAeC,EAAmBC,EAC5B,CAjDR,IAAAC,EAkDEF,EAAY,IAAI,OAAOA,EAAW,GAAG,EAGrC,IAAIG,EACAC,EAAQ,EACZ,EAAG,CACDD,EAAQH,EAAU,KAAKD,CAAK,EAG5B,IAAMM,GAAQH,EAAAC,GAAA,YAAAA,EAAO,QAAP,KAAAD,EAAgBH,EAAM,OAKpC,GAJIK,EAAQC,GACVJ,EAAGG,EAAOC,CAAK,EAGbF,EAAO,CACT,GAAM,CAACG,CAAI,EAAIH,EACfC,EAAQD,EAAM,MAAQG,EAAK,OAGvBA,EAAK,SAAW,IAClBN,EAAU,UAAYG,EAAM,MAAQ,EACxC,CACF,OAASA,EACX,CCFO,SAASI,GACdC,EAAeC,EACT,CAEN,IAAIC,EAAQ,EACRC,EAAQ,EACRC,EAAM,EAGV,QAASC,EAAQ,EAAGD,EAAMJ,EAAM,OAAQI,IAGlCJ,EAAM,OAAOI,CAAG,IAAM,KAAOA,EAAMD,EACrCF,EAAGC,EAAO,EAAcC,EAAOA,EAAQC,CAAG,EAGjCJ,EAAM,OAAOI,CAAG,IAAM,MAC3BJ,EAAM,OAAOG,EAAQ,CAAC,IAAM,IAC1B,EAAEE,IAAU,GACdJ,EAAGC,IAAS,EAAmBC,EAAOC,EAAM,CAAC,EAGtCJ,EAAM,OAAOI,EAAM,CAAC,IAAM,KAC/BC,MAAY,GACdJ,EAAGC,EAAO,EAAkBC,EAAOC,EAAM,CAAC,EAI9CD,EAAQC,EAAM,GAKdA,EAAMD,GACRF,EAAGC,EAAO,EAAcC,EAAOC,CAAG,CACtC,CCnDO,SAASE,GACdC,EAAeC,EAAsBC,EAAuBC,EAAO,GAC3D,CACR,OAAOC,EAAa,CAACJ,CAAK,EAAGC,EAAOC,EAAWC,CAAI,EAAE,IAAI,CAC3D,CAYO,SAASC,EACdC,EAAkBJ,EAAsBC,EAAuBC,EAAO,GAC5D,CAGV,IAAMG,EAAU,CAAC,CAAC,EAClB,QAASC,EAAI,EAAGA,EAAIN,EAAM,OAAQM,IAAK,CACrC,IAAMC,EAAOP,EAAMM,EAAI,CAAC,EAClBE,EAAOR,EAAMM,CAAC,EAGdG,EAAIF,EAAKA,EAAK,OAAS,CAAC,IAAM,EAAI,KAClCG,EAAIF,EAAK,CAAC,IAAoB,GAGpCH,EAAQ,KAAK,EAAEI,EAAIC,GAAKL,EAAQA,EAAQ,OAAS,CAAC,CAAC,CACrD,CAGA,OAAOD,EAAO,IAAI,CAACL,EAAOY,IAAM,CAC9B,IAAIC,EAAS,EAGPC,EAAS,IAAI,IACnB,QAAWJ,KAAKR,EAAU,KAAK,CAACa,EAAGC,IAAMD,EAAIC,CAAC,EAAG,CAC/C,IAAMC,EAAQP,EAAI,QACZQ,EAAQR,IAAM,GACpB,GAAIJ,EAAQY,CAAK,IAAMN,EACrB,SAGF,IAAIO,EAAQL,EAAO,IAAII,CAAK,EACxB,OAAOC,GAAU,aACnBL,EAAO,IAAII,EAAOC,EAAQ,CAAC,CAAC,EAG9BA,EAAM,KAAKF,CAAK,CAClB,CAGA,GAAIH,EAAO,OAAS,EAClB,OAAOd,EAGT,IAAMoB,EAAmB,CAAC,EAC1B,OAAW,CAACF,EAAOG,CAAO,IAAKP,EAAQ,CACrC,IAAMP,EAAIN,EAAMiB,CAAK,EAGfI,EAASf,EAAE,CAAC,IAAiB,GAC7BgB,EAAShB,EAAEA,EAAE,OAAS,CAAC,IAAM,GAC7BiB,EAASjB,EAAEA,EAAE,OAAS,CAAC,IAAM,EAAI,KAGnCJ,GAAQmB,EAAQT,GAClBO,EAAO,KAAKpB,EAAM,MAAMa,EAAQS,CAAK,CAAC,EAGxC,IAAIG,EAAQzB,EAAM,MAAMsB,EAAOC,EAAMC,CAAM,EAC3C,QAAWE,KAAKL,EAAQ,KAAK,CAACN,EAAGC,IAAMA,EAAID,CAAC,EAAG,CAG7C,IAAML,GAAKH,EAAEmB,CAAC,IAAM,IAAMJ,EACpBX,GAAKJ,EAAEmB,CAAC,IAAM,EAAI,MAAShB,EAGjCe,EAAQ,CACNA,EAAM,MAAM,EAAGf,CAAC,EAChB,SACAe,EAAM,MAAMf,EAAGC,CAAC,EAChB,UACAc,EAAM,MAAMd,CAAC,CACf,EAAE,KAAK,EAAE,CACX,CAMA,GAHAE,EAASU,EAAMC,EAGXJ,EAAO,KAAKK,CAAK,IAAM,EACzB,KACJ,CAGA,OAAItB,GAAQU,EAASb,EAAM,QACzBoB,EAAO,KAAKpB,EAAM,MAAMa,CAAM,CAAC,EAG1BO,EAAO,KAAK,EAAE,CACvB,CAAC,CACH,CChHO,SAASO,GACdC,EACc,CACd,IAAMC,EAAuB,CAAC,EAC9B,GAAI,OAAOD,GAAU,YACnB,OAAOC,EAGT,IAAMC,EAAS,MAAM,QAAQF,CAAK,EAAIA,EAAQ,CAACA,CAAK,EACpD,QAASG,EAAI,EAAGA,EAAID,EAAO,OAAQC,IAAK,CACtC,IAAMC,EAAQ,KAAK,UAAU,MACvBC,EAAQD,EAAM,OAGpBE,GAAQJ,EAAOC,CAAC,EAAG,CAACI,EAAOC,EAAMC,EAAOC,IAAQ,CA/DpD,IAAAC,EAiEM,OADAP,EAAAO,EAAMJ,GAASF,KAAfD,EAAAO,GAA0B,CAAC,GACnBH,EAAM,CAGZ,OACA,OACEJ,EAAMG,CAAK,EAAE,KACXE,GAAe,GACfC,EAAMD,GAAU,EAChBD,CACF,EACA,MAGF,OACE,IAAMI,EAAUV,EAAOC,CAAC,EAAE,MAAMM,EAAOC,CAAG,EAC1CG,EAAMD,EAAS,KAAK,UAAU,UAAW,CAACE,EAAOC,IAAU,CAOzD,GAAI,OAAO,KAAK,WAAc,YAAa,CACzC,IAAMC,EAAaJ,EAAQ,MAAME,EAAOC,CAAK,EAC7C,GAAI,WAAW,KAAK,KAAK,UAAU,OAAOC,CAAU,CAAC,EAAG,CACtD,IAAMC,EAAW,KAAK,UAAU,QAAQD,CAAU,EAClD,QAASE,EAAI,EAAGC,EAAI,EAAGD,EAAID,EAAS,OAAQC,IAG1Cd,EAAAG,KAAAH,EAAAG,GAAiB,CAAC,GAClBH,EAAMG,CAAK,EAAE,KACXE,EAAQK,EAAQK,GAAM,GACtBF,EAASC,CAAC,EAAE,QAAW,EACvBV,CACF,EAGAP,EAAO,KAAK,IAAI,KAAK,MACnBgB,EAASC,CAAC,EAAE,YAAY,EAAG,CACzB,SAAUX,GAAS,GAAKH,EAAMG,CAAK,EAAE,OAAS,CAChD,CACF,CAAC,EAGDY,GAAKF,EAASC,CAAC,EAAE,OAEnB,MACF,CACF,CAGAd,EAAMG,CAAK,EAAE,KACXE,EAAQK,GAAS,GACjBC,EAAQD,GAAU,EAClBN,CACF,EAGAP,EAAO,KAAK,IAAI,KAAK,MACnBW,EAAQ,MAAME,EAAOC,CAAK,EAAE,YAAY,EAAG,CACzC,SAAUR,GAAS,GAAKH,EAAMG,CAAK,EAAE,OAAS,CAChD,CACF,CAAC,CACH,CAAC,CACL,CACF,CAAC,CACH,CAGA,OAAON,CACT,CCjEO,SAASmB,GACdC,EAAeC,EAAgBC,GAAQA,EAC/B,CACR,OAAOF,EAGJ,KAAK,EAGL,MAAM,YAAY,EAChB,IAAI,CAACG,EAAOC,IAAUA,EAAQ,EAC3BD,EAAM,QAAQ,+BAAgC,IAAI,EAClDA,CACJ,EACC,KAAK,EAAE,EAGT,QAAQ,kCAAmC,EAAE,EAG7C,MAAM,MAAM,EACV,OAAO,CAACE,EAAMH,IAAS,CACtB,IAAMI,EAAOL,EAAGC,CAAI,EACpB,MAAO,CAAC,GAAGG,EAAM,GAAG,MAAM,QAAQC,CAAI,EAAIA,EAAO,CAACA,CAAI,CAAC,CACzD,EAAG,CAAC,CAAa,EAChB,IAAIJ,GAAQ,UAAU,KAAKA,CAAI,EAAI,GAAGA,CAAI,IAAMA,CAAI,EACpD,IAAIA,GAAQ,mBAAmB,KAAKA,CAAI,EAAIA,EAAO,GAAGA,CAAI,GAAG,EAC7D,KAAK,GAAG,CACf,CCxCO,SAASK,GACdC,EACQ,CAGR,OAAOC,GAAUD,EAAOE,GAAQ,CAC9B,IAAMC,EAAkB,CAAC,EAGnBC,EAAQ,IAAI,KAAK,WAAWF,CAAI,EACtCE,EAAM,IAAI,EAGV,OAAW,CAAE,KAAAC,EAAM,IAAKC,EAAM,MAAAC,EAAO,IAAAC,CAAI,IAAKJ,EAAM,QAClD,OAAQC,EAAM,CAGZ,IAAK,QACE,CAAC,QAAS,OAAQ,MAAM,EAAE,SAASC,CAAI,IAC1CJ,EAAO,CACLA,EAAK,MAAM,EAAGM,CAAG,EACjB,IACAN,EAAK,MAAMM,EAAM,CAAC,CACpB,EAAE,KAAK,EAAE,GACX,MAGF,IAAK,OACHC,EAAMH,EAAM,KAAK,UAAU,UAAW,IAAII,IAAU,CAClDP,EAAM,KAAK,CACTD,EAAK,MAAM,EAAGK,CAAK,EACnBD,EAAK,MAAM,GAAGI,CAAK,EACnBR,EAAK,MAAMM,CAAG,CAChB,EAAE,KAAK,EAAE,CAAC,CACZ,CAAC,CACL,CAGF,OAAOL,CACT,CAAC,CACH,CAgBO,SAASQ,GACdC,EACqB,CACrB,IAAMZ,EAAS,IAAI,KAAK,MAAM,CAAC,QAAS,OAAQ,MAAM,CAAC,EACxC,IAAI,KAAK,YAAYY,EAAOZ,CAAK,EAGzC,MAAM,EACb,QAAWa,KAAUb,EAAM,QACzBa,EAAO,YAAc,GAGjBA,EAAO,KAAK,WAAW,GAAG,IAC5BA,EAAO,SAAW,KAAK,MAAM,SAAS,QACtCA,EAAO,KAAOA,EAAO,KAAK,MAAM,CAAC,GAI/BA,EAAO,KAAK,SAAS,GAAG,IAC1BA,EAAO,SAAW,KAAK,MAAM,SAAS,SACtCA,EAAO,KAAOA,EAAO,KAAK,MAAM,EAAG,EAAE,GAKzC,OAAOb,EAAM,OACf,CAUO,SAASc,GACdd,EAA4BG,EACV,CAxJpB,IAAAY,EAyJE,IAAMC,EAAU,IAAI,IAAuBhB,CAAK,EAG1CiB,EAA2B,CAAC,EAClC,QAASC,EAAI,EAAGA,EAAIf,EAAM,OAAQe,IAChC,QAAWL,KAAUG,EACfb,EAAMe,CAAC,EAAE,WAAWL,EAAO,IAAI,IACjCI,EAAOJ,EAAO,IAAI,EAAI,GACtBG,EAAQ,OAAOH,CAAM,GAI3B,QAAWA,KAAUG,GACfD,EAAA,KAAK,iBAAL,MAAAA,EAAA,UAAsBF,EAAO,QAC/BI,EAAOJ,EAAO,IAAI,EAAI,IAG1B,OAAOI,CACT,CClIO,SAASE,GACdC,EAAeC,EACG,CAClB,IAAMC,EAAW,IAAI,IAGfC,EAAW,IAAI,YAAYH,EAAM,MAAM,EAC7C,QAASI,EAAI,EAAGA,EAAIJ,EAAM,OAAQI,IAChC,QAASC,EAAID,EAAI,EAAGC,EAAIL,EAAM,OAAQK,IACtBL,EAAM,MAAMI,EAAGC,CAAC,IACjBJ,IACXE,EAASC,CAAC,EAAIC,EAAID,GAIxB,IAAME,EAAQ,CAAC,CAAC,EAChB,QAAS,EAAIA,EAAM,OAAQ,EAAI,GAAI,CACjC,IAAMC,EAAID,EAAM,EAAE,CAAC,EACnB,QAASE,EAAI,EAAGA,EAAIL,EAASI,CAAC,EAAGC,IAC3BL,EAASI,EAAIC,CAAC,EAAIL,EAASI,CAAC,EAAIC,IAClCN,EAAS,IAAIF,EAAM,MAAMO,EAAGA,EAAIC,CAAC,CAAC,EAClCF,EAAM,GAAG,EAAIC,EAAIC,GAIrB,IAAMA,EAAID,EAAIJ,EAASI,CAAC,EACpBJ,EAASK,CAAC,GAAKA,EAAIR,EAAM,OAAS,IACpCM,EAAM,GAAG,EAAIE,GAGfN,EAAS,IAAIF,EAAM,MAAMO,EAAGC,CAAC,CAAC,CAChC,CAGA,OAAIN,EAAS,IAAI,EAAE,EACV,IAAI,IAAI,CAACF,CAAK,CAAC,EAGjBE,CACT,CCJA,SAASO,GAAUC,EAAmC,CACpD,OAAQC,GACEC,GAAwB,CAC9B,GAAI,OAAOA,EAAID,CAAI,GAAM,YACvB,OAGF,IAAME,EAAK,CAACD,EAAI,SAAUD,CAAI,EAAE,KAAK,GAAG,EACxC,OAAAD,EAAM,IAAIG,EAAI,KAAK,UAAU,MAAQ,CAAC,CAAC,EAGhCD,EAAID,CAAI,CACjB,CAEJ,CAUA,SAASG,GAAWC,EAAaC,EAAuB,CACtD,GAAM,CAACC,EAAGC,CAAC,EAAI,CAAC,IAAI,IAAIH,CAAC,EAAG,IAAI,IAAIC,CAAC,CAAC,EACtC,MAAO,CACL,GAAG,IAAI,IAAI,CAAC,GAAGC,CAAC,EAAE,OAAOE,GAAS,CAACD,EAAE,IAAIC,CAAK,CAAC,CAAC,CAClD,CACF,CASO,IAAMC,EAAN,KAAa,CA2BX,YAAY,CAAE,OAAAC,EAAQ,KAAAC,EAAM,QAAAC,CAAQ,EAAgB,CACzD,IAAMC,EAAQf,GAAU,KAAK,MAAQ,IAAI,GAAK,EAG9C,KAAK,IAAMgB,GAAuBH,CAAI,EACtC,KAAK,QAAUC,EAGf,KAAK,MAAQ,KAAK,UAAY,CAC5B,KAAK,kBAAoB,CAAC,UAAU,EACpC,KAAK,EAAE,CAAC,EAGJF,EAAO,KAAK,SAAW,GAAKA,EAAO,KAAK,CAAC,IAAM,KAEjD,KAAK,IAAI,KAAKA,EAAO,KAAK,CAAC,CAAC,CAAC,EACpBA,EAAO,KAAK,OAAS,GAC9B,KAAK,IAAI,KAAK,cAAc,GAAGA,EAAO,IAAI,CAAC,EAI7C,KAAK,UAAYK,GACjB,KAAK,UAAU,UAAY,IAAI,OAAOL,EAAO,SAAS,EAGtD,KAAK,UAAY,kBAAmB,KAChC,IAAI,KAAK,cACT,OAGJ,IAAMM,EAAMb,GAAW,CACrB,UAAW,iBAAkB,SAC/B,EAAGO,EAAO,QAAQ,EAGlB,QAAWO,KAAQP,EAAO,KAAK,IAAIQ,GAEjCA,IAAa,KAAO,KAAO,KAAKA,CAAQ,CACzC,EACC,QAAWC,KAAMH,EACf,KAAK,SAAS,OAAOC,EAAKE,CAAE,CAAC,EAC7B,KAAK,eAAe,OAAOF,EAAKE,CAAE,CAAC,EAIvC,KAAK,IAAI,UAAU,EAGnB,KAAK,MAAM,QAAS,CAAE,MAAO,IAAK,UAAWN,EAAM,OAAO,CAAE,CAAC,EAC7D,KAAK,MAAM,OAAS,CAAE,MAAO,EAAK,UAAWA,EAAM,MAAM,CAAE,CAAC,EAC5D,KAAK,MAAM,OAAS,CAAE,MAAO,IAAK,UAAWA,EAAM,MAAM,CAAE,CAAC,EAG5D,QAAWZ,KAAOU,EAChB,KAAK,IAAIV,EAAK,CAAE,MAAOA,EAAI,KAAM,CAAC,CACtC,CAAC,CACH,CASO,OAAOmB,EAA6B,CAUzC,GAPAA,EAAQA,EAAM,QAAQ,WAAC,eAAY,IAAE,EAAEZ,GAC9B,CAAC,GAAGa,GAAQb,EAAO,KAAK,MAAM,aAAa,CAAC,EAChD,KAAK,IAAI,CACb,EAGDY,EAAQE,GAAqBF,CAAK,EAC9B,CAACA,EACH,MAAO,CAAE,MAAO,CAAC,CAAE,EAGrB,IAAMG,EAAUC,GAAiBJ,CAAK,EACnC,OAAOK,GACNA,EAAO,WAAa,KAAK,MAAM,SAAS,UACzC,EAGGC,EAAS,KAAK,MAAM,OAAON,CAAK,EAGnC,OAAqB,CAACO,EAAM,CAAE,IAAAC,EAAK,MAAAC,EAAO,UAAAC,CAAU,IAAM,CACzD,IAAI7B,EAAM,KAAK,IAAI,IAAI2B,CAAG,EAC1B,GAAI,OAAO3B,GAAQ,YAAa,CAG9BA,EAAM8B,EAAA,GAAK9B,GACPA,EAAI,OACNA,EAAI,KAAO,CAAC,GAAGA,EAAI,IAAI,GAGzB,IAAM+B,EAAQC,GACZV,EACA,OAAO,KAAKO,EAAU,QAAQ,CAChC,EAGA,QAAWjB,KAAS,KAAK,MAAM,OAAQ,CACrC,GAAI,OAAOZ,EAAIY,CAAK,GAAM,YACxB,SAGF,IAAMqB,EAAwB,CAAC,EAC/B,QAAWC,KAAS,OAAO,OAAOL,EAAU,QAAQ,EAC9C,OAAOK,EAAMtB,CAAK,GAAM,aAC1BqB,EAAU,KAAK,GAAGC,EAAMtB,CAAK,EAAE,QAAQ,EAG3C,GAAI,CAACqB,EAAU,OACb,SAGF,IAAMnC,EAAQ,KAAK,MAAM,IAAI,CAACE,EAAI,SAAUY,CAAK,EAAE,KAAK,GAAG,CAAC,EACtDM,EAAK,MAAM,QAAQlB,EAAIY,CAAK,CAAC,EAC/BuB,EACAC,GAGJpC,EAAIY,CAAK,EAAIM,EAAGlB,EAAIY,CAAK,EAAGd,EAAOmC,EAAWrB,IAAU,MAAM,CAChE,CAGA,IAAMyB,EAAQ,CAAC,CAACrC,EAAI,OAClB,OAAO,OAAO+B,CAAK,EAChB,OAAOO,GAAKA,CAAC,EAAE,OAClB,OAAO,KAAKP,CAAK,EAAE,OAGrBL,EAAK,KAAKa,EAAAT,EAAA,GACL9B,GADK,CAER,MAAO4B,GAAS,EAAIY,EAAAH,EAAS,IAC7B,MAAAN,CACF,EAAC,CACH,CACA,OAAOL,CACT,EAAG,CAAC,CAAC,EAGJ,KAAK,CAACvB,EAAGC,IAAMA,EAAE,MAAQD,EAAE,KAAK,EAGhC,OAAO,CAACsC,EAAOC,IAAW,CACzB,IAAM1C,EAAM,KAAK,IAAI,IAAI0C,EAAO,QAAQ,EACxC,GAAI,OAAO1C,GAAQ,YAAa,CAC9B,IAAM2B,EAAM3B,EAAI,OACZA,EAAI,OAAO,SACXA,EAAI,SACRyC,EAAM,IAAId,EAAK,CAAC,GAAGc,EAAM,IAAId,CAAG,GAAK,CAAC,EAAGe,CAAM,CAAC,CAClD,CACA,OAAOD,CACT,EAAG,IAAI,GAA2B,EAGpC,OAAW,CAACd,EAAKc,CAAK,IAAKhB,EACzB,GAAI,CAACgB,EAAM,KAAKf,GAAQA,EAAK,WAAaC,CAAG,EAAG,CAC9C,IAAM3B,EAAM,KAAK,IAAI,IAAI2B,CAAG,EAC5Bc,EAAM,KAAKF,EAAAT,EAAA,GAAK9B,GAAL,CAAU,MAAO,EAAG,MAAO,CAAC,CAAE,EAAC,CAC5C,CAGF,IAAI2C,EACJ,GAAI,KAAK,QAAQ,QAAS,CACxB,IAAMC,EAAS,KAAK,MAAM,MAAMC,GAAW,CACzC,QAAWrB,KAAUF,EACnBuB,EAAQ,KAAKrB,EAAO,KAAM,CACxB,OAAQ,CAAC,OAAO,EAChB,SAAU,KAAK,MAAM,SAAS,SAC9B,SAAU,KAAK,MAAM,SAAS,QAChC,CAAC,CACL,CAAC,EAGDmB,EAAUC,EAAO,OACb,OAAO,KAAKA,EAAO,CAAC,EAAE,UAAU,QAAQ,EACxC,CAAC,CACP,CAGA,OAAOd,EAAA,CACL,MAAO,CAAC,GAAGL,EAAO,OAAO,CAAC,GACvB,OAAOkB,GAAY,aAAe,CAAE,QAAAA,CAAQ,EAEnD,CACF,EX5QA,IAAIG,GAqBJ,SAAeC,GACbC,EACe,QAAAC,EAAA,sBACf,IAAIC,EAAO,UAGX,GAAI,OAAO,QAAW,aAAe,iBAAkB,OAAQ,CAC7D,IAAMC,EAASC,GAA8B,aAAa,EACpD,CAACC,CAAI,EAAIF,EAAO,IAAI,MAAM,SAAS,EAGzCD,EAAOA,EAAK,QAAQ,KAAMG,CAAI,CAChC,CAGA,IAAMC,EAAU,CAAC,EACjB,QAAWC,KAAQP,EAAO,KAAM,CAC9B,OAAQO,EAAM,CAGZ,IAAK,KACHD,EAAQ,KAAK,GAAGJ,CAAI,aAAa,EACjC,MAGF,IAAK,KACL,IAAK,KACHI,EAAQ,KAAK,GAAGJ,CAAI,aAAa,EACjC,KACJ,CAGIK,IAAS,MACXD,EAAQ,KAAK,GAAGJ,CAAI,aAAaK,CAAI,SAAS,CAClD,CAGIP,EAAO,KAAK,OAAS,GACvBM,EAAQ,KAAK,GAAGJ,CAAI,wBAAwB,EAG1CI,EAAQ,SACV,MAAM,cACJ,GAAGJ,CAAI,mCACP,GAAGI,CACL,EACJ,GAaA,SAAsBE,GACpBC,EACwB,QAAAR,EAAA,sBACxB,OAAQQ,EAAQ,KAAM,CAGpB,OACE,aAAMV,GAAqBU,EAAQ,KAAK,MAAM,EAC9CX,GAAQ,IAAIY,EAAOD,EAAQ,IAAI,EACxB,CACL,MACF,EAGF,OACE,IAAME,EAAQF,EAAQ,KACtB,GAAI,CACF,MAAO,CACL,OACA,KAAMX,GAAM,OAAOa,CAAK,CAC1B,CAGF,OAASC,EAAK,CACZ,eAAQ,KAAK,kBAAkBD,CAAK,oCAA+B,EACnE,QAAQ,KAAKC,CAAG,EACT,CACL,OACA,KAAM,CAAE,MAAO,CAAC,CAAE,CACpB,CACF,CAGF,QACE,MAAM,IAAI,UAAU,sBAAsB,CAC9C,CACF,GAOA,KAAK,KAAO,GAAAC,QAGZ,iBAAiB,UAAiBC,GAAMb,EAAA,wBACtC,YAAY,MAAMO,GAAQM,EAAG,IAAI,CAAC,CACpC,EAAC",
+  "names": ["require_lunr", "__commonJSMin", "exports", "module", "lunr", "config", "builder", "global", "message", "obj", "clone", "keys", "key", "val", "docRef", "fieldName", "stringValue", "s", "n", "fieldRef", "elements", "i", "other", "object", "a", "b", "intersection", "element", "posting", "documentCount", "documentsWithTerm", "x", "str", "metadata", "fn", "t", "len", "tokens", "sliceEnd", "sliceStart", "char", "sliceLength", "tokenMetadata", "label", "isRegistered", "serialised", "pipeline", "fnName", "fns", "existingFn", "newFn", "pos", "stackLength", "memo", "j", "result", "k", "token", "index", "start", "end", "pivotPoint", "pivotIndex", "insertIdx", "position", "sumOfSquares", "elementsLength", "otherVector", "dotProduct", "aLen", "bLen", "aVal", "bVal", "output", "step2list", "step3list", "c", "v", "C", "V", "mgr0", "meq1", "mgr1", "s_v", "re_mgr0", "re_mgr1", "re_meq1", "re_s_v", "re_1a", "re2_1a", "re_1b", "re2_1b", "re_1b_2", "re2_1b_2", "re3_1b_2", "re4_1b_2", "re_1c", "re_2", "re_3", "re_4", "re2_4", "re_5", "re_5_1", "re3_5", "porterStemmer", "w", "stem", "suffix", "firstch", "re", "re2", "re3", "re4", "fp", "stopWords", "words", "stopWord", "arr", "clause", "editDistance", "root", "stack", "frame", "noEditNode", "insertionNode", "substitutionNode", "charA", "charB", "transposeNode", "node", "final", "next", "edges", "edge", "labels", "qEdges", "qLen", "nEdges", "nLen", "q", "qEdge", "nEdge", "qNode", "word", "commonPrefix", "nextNode", "downTo", "childKey", "attrs", "queryString", "query", "parser", "matchingFields", "queryVectors", "termFieldCache", "requiredMatches", "prohibitedMatches", "terms", "clauseMatches", "m", "term", "termTokenSet", "expandedTerms", "field", "expandedTerm", "termIndex", "fieldPosting", "matchingDocumentRefs", "termField", "matchingDocumentsSet", "l", "matchingDocumentRef", "matchingFieldRef", "fieldMatch", "allRequiredMatches", "allProhibitedMatches", "matchingFieldRefs", "results", "matches", "fieldVector", "score", "docMatch", "match", "invertedIndex", "fieldVectors", "ref", "serializedIndex", "serializedVectors", "serializedInvertedIndex", "tokenSetBuilder", "tuple", "attributes", "number", "doc", "fields", "extractor", "fieldTerms", "metadataKey", "fieldRefs", "numberOfFields", "accumulator", "documentsWithField", "fieldRefsLength", "termIdfCache", "fieldLength", "termFrequencies", "termsLength", "fieldBoost", "docBoost", "tf", "idf", "scoreWithPrecision", "args", "clonedMetadata", "metadataKeys", "otherMatchData", "allFields", "options", "state", "subSlices", "type", "charCode", "lexer", "lexeme", "completedClause", "errorMessage", "nextLexeme", "possibleFields", "f", "boost", "factory", "import_lunr", "getElement", "selector", "node", "el", "getOptionalElement", "obj", "data", "key", "x", "y", "nodes", "parent", "i", "node", "setupSearchDocumentMap", "docs", "map", "doc", "path", "article", "split", "input", "separator", "fn", "_a", "match", "index", "until", "term", "extract", "input", "fn", "block", "start", "end", "stack", "highlight", "input", "table", "positions", "full", "highlightAll", "inputs", "mapping", "t", "prev", "next", "p", "q", "i", "cursor", "blocks", "a", "b", "index", "block", "group", "slices", "indexes", "start", "end", "length", "slice", "j", "tokenize", "input", "tokens", "inputs", "i", "table", "total", "extract", "block", "type", "start", "end", "_a", "section", "split", "index", "until", "subsection", "segments", "s", "l", "transform", "query", "fn", "term", "parts", "index", "prev", "next", "transformSearchQuery", "query", "transform", "part", "terms", "lexer", "type", "term", "start", "end", "split", "range", "parseSearchQuery", "value", "clause", "getSearchQueryTerms", "_a", "clauses", "result", "t", "segment", "query", "index", "segments", "wordcuts", "i", "j", "stack", "p", "q", "extractor", "table", "name", "doc", "id", "difference", "a", "b", "x", "y", "value", "Search", "config", "docs", "options", "field", "setupSearchDocumentMap", "tokenize", "fns", "lang", "language", "fn", "query", "segment", "transformSearchQuery", "clauses", "parseSearchQuery", "clause", "groups", "item", "ref", "score", "matchData", "__spreadValues", "terms", "getSearchQueryTerms", "positions", "match", "highlightAll", "highlight", "boost", "t", "__spreadProps", "__pow", "items", "result", "suggest", "titles", "builder", "index", "setupSearchLanguages", "config", "__async", "base", "worker", "getElement", "path", "scripts", "lang", "handler", "message", "Search", "query", "err", "lunr", "ev"]
+}
diff --git a/assets/stylesheets/main.7e359304.min.css b/assets/stylesheets/main.7e359304.min.css
new file mode 100644
index 000000000..e5a7e3a73
--- /dev/null
+++ b/assets/stylesheets/main.7e359304.min.css
@@ -0,0 +1 @@
+@charset "UTF-8";html{-webkit-text-size-adjust:none;-moz-text-size-adjust:none;text-size-adjust:none;box-sizing:border-box}*,:after,:before{box-sizing:inherit}@media (prefers-reduced-motion){*,:after,:before{transition:none!important}}body{margin:0}a,button,input,label{-webkit-tap-highlight-color:transparent}a{color:inherit;text-decoration:none}hr{border:0;box-sizing:initial;display:block;height:.05rem;overflow:visible;padding:0}small{font-size:80%}sub,sup{line-height:1em}img{border-style:none}table{border-collapse:initial;border-spacing:0}td,th{font-weight:400;vertical-align:top}button{background:#0000;border:0;font-family:inherit;font-size:inherit;margin:0;padding:0}input{border:0;outline:none}:root{--md-primary-fg-color:#4051b5;--md-primary-fg-color--light:#5d6cc0;--md-primary-fg-color--dark:#303fa1;--md-primary-bg-color:#fff;--md-primary-bg-color--light:#ffffffb3;--md-accent-fg-color:#526cfe;--md-accent-fg-color--transparent:#526cfe1a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-scheme=default]{color-scheme:light}[data-md-color-scheme=default] img[src$="#gh-dark-mode-only"],[data-md-color-scheme=default] img[src$="#only-dark"]{display:none}:root,[data-md-color-scheme=default]{--md-hue:225deg;--md-default-fg-color:#000000de;--md-default-fg-color--light:#0000008a;--md-default-fg-color--lighter:#00000052;--md-default-fg-color--lightest:#00000012;--md-default-bg-color:#fff;--md-default-bg-color--light:#ffffffb3;--md-default-bg-color--lighter:#ffffff4d;--md-default-bg-color--lightest:#ffffff1f;--md-code-fg-color:#36464e;--md-code-bg-color:#f5f5f5;--md-code-hl-color:#4287ff;--md-code-hl-color--light:#4287ff1a;--md-code-hl-number-color:#d52a2a;--md-code-hl-special-color:#db1457;--md-code-hl-function-color:#a846b9;--md-code-hl-constant-color:#6e59d9;--md-code-hl-keyword-color:#3f6ec6;--md-code-hl-string-color:#1c7d4d;--md-code-hl-name-color:var(--md-code-fg-color);--md-code-hl-operator-color:var(--md-default-fg-color--light);--md-code-hl-punctuation-color:var(--md-default-fg-color--light);--md-code-hl-comment-color:var(--md-default-fg-color--light);--md-code-hl-generic-color:var(--md-default-fg-color--light);--md-code-hl-variable-color:var(--md-default-fg-color--light);--md-typeset-color:var(--md-default-fg-color);--md-typeset-a-color:var(--md-primary-fg-color);--md-typeset-del-color:#f5503d26;--md-typeset-ins-color:#0bd57026;--md-typeset-kbd-color:#fafafa;--md-typeset-kbd-accent-color:#fff;--md-typeset-kbd-border-color:#b8b8b8;--md-typeset-mark-color:#ffff0080;--md-typeset-table-color:#0000001f;--md-typeset-table-color--light:rgba(0,0,0,.035);--md-admonition-fg-color:var(--md-default-fg-color);--md-admonition-bg-color:var(--md-default-bg-color);--md-warning-fg-color:#000000de;--md-warning-bg-color:#ff9;--md-footer-fg-color:#fff;--md-footer-fg-color--light:#ffffffb3;--md-footer-fg-color--lighter:#ffffff73;--md-footer-bg-color:#000000de;--md-footer-bg-color--dark:#00000052;--md-shadow-z1:0 0.2rem 0.5rem #0000000d,0 0 0.05rem #0000001a;--md-shadow-z2:0 0.2rem 0.5rem #0000001a,0 0 0.05rem #00000040;--md-shadow-z3:0 0.2rem 0.5rem #0003,0 0 0.05rem #00000059}.md-icon svg{fill:currentcolor;display:block;height:1.2rem;width:1.2rem}body{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;--md-text-font-family:var(--md-text-font,_),-apple-system,BlinkMacSystemFont,Helvetica,Arial,sans-serif;--md-code-font-family:var(--md-code-font,_),SFMono-Regular,Consolas,Menlo,monospace}aside,body,input{font-feature-settings:"kern","liga";color:var(--md-typeset-color);font-family:var(--md-text-font-family)}code,kbd,pre{font-feature-settings:"kern";font-family:var(--md-code-font-family)}:root{--md-typeset-table-sort-icon:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m18 21-4-4h3V7h-3l4-4 4 4h-3v10h3M2 19v-2h10v2M2 13v-2h7v2M2 7V5h4v2H2Z"/></svg>');--md-typeset-table-sort-icon--asc:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 17h3l-4 4-4-4h3V3h2M2 17h10v2H2M6 5v2H2V5m0 6h7v2H2v-2Z"/></svg>');--md-typeset-table-sort-icon--desc:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 7h3l-4-4-4 4h3v14h2M2 17h10v2H2M6 5v2H2V5m0 6h7v2H2v-2Z"/></svg>')}.md-typeset{-webkit-print-color-adjust:exact;color-adjust:exact;font-size:.8rem;line-height:1.6}@media print{.md-typeset{font-size:.68rem}}.md-typeset blockquote,.md-typeset dl,.md-typeset figure,.md-typeset ol,.md-typeset pre,.md-typeset ul{margin-bottom:1em;margin-top:1em}.md-typeset h1{color:var(--md-default-fg-color--light);font-size:2em;line-height:1.3;margin:0 0 1.25em}.md-typeset h1,.md-typeset h2{font-weight:300;letter-spacing:-.01em}.md-typeset h2{font-size:1.5625em;line-height:1.4;margin:1.6em 0 .64em}.md-typeset h3{font-size:1.25em;font-weight:400;letter-spacing:-.01em;line-height:1.5;margin:1.6em 0 .8em}.md-typeset h2+h3{margin-top:.8em}.md-typeset h4{font-weight:700;letter-spacing:-.01em;margin:1em 0}.md-typeset h5,.md-typeset h6{color:var(--md-default-fg-color--light);font-size:.8em;font-weight:700;letter-spacing:-.01em;margin:1.25em 0}.md-typeset h5{text-transform:uppercase}.md-typeset hr{border-bottom:.05rem solid var(--md-default-fg-color--lightest);display:flow-root;margin:1.5em 0}.md-typeset a{color:var(--md-typeset-a-color);word-break:break-word}.md-typeset a,.md-typeset a:before{transition:color 125ms}.md-typeset a:focus,.md-typeset a:hover{color:var(--md-accent-fg-color)}.md-typeset a:focus code,.md-typeset a:hover code{background-color:var(--md-accent-fg-color--transparent)}.md-typeset a code{color:currentcolor;transition:background-color 125ms}.md-typeset a.focus-visible{outline-color:var(--md-accent-fg-color);outline-offset:.2rem}.md-typeset code,.md-typeset kbd,.md-typeset pre{color:var(--md-code-fg-color);direction:ltr;font-variant-ligatures:none}@media print{.md-typeset code,.md-typeset kbd,.md-typeset pre{white-space:pre-wrap}}.md-typeset code{background-color:var(--md-code-bg-color);border-radius:.1rem;-webkit-box-decoration-break:clone;box-decoration-break:clone;font-size:.85em;padding:0 .2941176471em;word-break:break-word}.md-typeset code:not(.focus-visible){-webkit-tap-highlight-color:transparent;outline:none}.md-typeset pre{display:flow-root;line-height:1.4;position:relative}.md-typeset pre>code{-webkit-box-decoration-break:slice;box-decoration-break:slice;box-shadow:none;display:block;margin:0;outline-color:var(--md-accent-fg-color);overflow:auto;padding:.7720588235em 1.1764705882em;scrollbar-color:var(--md-default-fg-color--lighter) #0000;scrollbar-width:thin;touch-action:auto;word-break:normal}.md-typeset pre>code:hover{scrollbar-color:var(--md-accent-fg-color) #0000}.md-typeset pre>code::-webkit-scrollbar{height:.2rem;width:.2rem}.md-typeset pre>code::-webkit-scrollbar-thumb{background-color:var(--md-default-fg-color--lighter)}.md-typeset pre>code::-webkit-scrollbar-thumb:hover{background-color:var(--md-accent-fg-color)}.md-typeset kbd{background-color:var(--md-typeset-kbd-color);border-radius:.1rem;box-shadow:0 .1rem 0 .05rem var(--md-typeset-kbd-border-color),0 .1rem 0 var(--md-typeset-kbd-border-color),0 -.1rem .2rem var(--md-typeset-kbd-accent-color) inset;color:var(--md-default-fg-color);display:inline-block;font-size:.75em;padding:0 .6666666667em;vertical-align:text-top;word-break:break-word}.md-typeset mark{background-color:var(--md-typeset-mark-color);-webkit-box-decoration-break:clone;box-decoration-break:clone;color:inherit;word-break:break-word}.md-typeset abbr{border-bottom:.05rem dotted var(--md-default-fg-color--light);cursor:help;text-decoration:none}.md-typeset small{opacity:.75}[dir=ltr] .md-typeset sub,[dir=ltr] .md-typeset sup{margin-left:.078125em}[dir=rtl] .md-typeset sub,[dir=rtl] .md-typeset sup{margin-right:.078125em}[dir=ltr] .md-typeset blockquote{padding-left:.6rem}[dir=rtl] .md-typeset blockquote{padding-right:.6rem}[dir=ltr] .md-typeset blockquote{border-left:.2rem solid var(--md-default-fg-color--lighter)}[dir=rtl] .md-typeset blockquote{border-right:.2rem solid var(--md-default-fg-color--lighter)}.md-typeset blockquote{color:var(--md-default-fg-color--light);margin-left:0;margin-right:0}.md-typeset ul{list-style-type:disc}[dir=ltr] .md-typeset ol,[dir=ltr] .md-typeset ul{margin-left:.625em}[dir=rtl] .md-typeset ol,[dir=rtl] .md-typeset ul{margin-right:.625em}.md-typeset ol,.md-typeset ul{padding:0}.md-typeset ol:not([hidden]),.md-typeset ul:not([hidden]){display:flow-root}.md-typeset ol ol,.md-typeset ul ol{list-style-type:lower-alpha}.md-typeset ol ol ol,.md-typeset ul ol ol{list-style-type:lower-roman}[dir=ltr] .md-typeset ol li,[dir=ltr] .md-typeset ul li{margin-left:1.25em}[dir=rtl] .md-typeset ol li,[dir=rtl] .md-typeset ul li{margin-right:1.25em}.md-typeset ol li,.md-typeset ul li{margin-bottom:.5em}.md-typeset ol li blockquote,.md-typeset ol li p,.md-typeset ul li blockquote,.md-typeset ul li p{margin:.5em 0}.md-typeset ol li:last-child,.md-typeset ul li:last-child{margin-bottom:0}[dir=ltr] .md-typeset ol li ol,[dir=ltr] .md-typeset ol li ul,[dir=ltr] .md-typeset ul li ol,[dir=ltr] .md-typeset ul li ul{margin-left:.625em}[dir=rtl] .md-typeset ol li ol,[dir=rtl] .md-typeset ol li ul,[dir=rtl] .md-typeset ul li ol,[dir=rtl] .md-typeset ul li ul{margin-right:.625em}.md-typeset ol li ol,.md-typeset ol li ul,.md-typeset ul li ol,.md-typeset ul li ul{margin-bottom:.5em;margin-top:.5em}[dir=ltr] .md-typeset dd{margin-left:1.875em}[dir=rtl] .md-typeset dd{margin-right:1.875em}.md-typeset dd{margin-bottom:1.5em;margin-top:1em}.md-typeset img,.md-typeset svg,.md-typeset video{height:auto;max-width:100%}.md-typeset img[align=left]{margin:1em 1em 1em 0}.md-typeset img[align=right]{margin:1em 0 1em 1em}.md-typeset img[align]:only-child{margin-top:0}.md-typeset figure{display:flow-root;margin:1em auto;max-width:100%;text-align:center;width:-webkit-fit-content;width:-moz-fit-content;width:fit-content}.md-typeset figure img{display:block;margin:0 auto}.md-typeset figcaption{font-style:italic;margin:1em auto;max-width:24rem}.md-typeset iframe{max-width:100%}.md-typeset table:not([class]){background-color:var(--md-default-bg-color);border:.05rem solid var(--md-typeset-table-color);border-radius:.1rem;display:inline-block;font-size:.64rem;max-width:100%;overflow:auto;touch-action:auto}@media print{.md-typeset table:not([class]){display:table}}.md-typeset table:not([class])+*{margin-top:1.5em}.md-typeset table:not([class]) td>:first-child,.md-typeset table:not([class]) th>:first-child{margin-top:0}.md-typeset table:not([class]) td>:last-child,.md-typeset table:not([class]) th>:last-child{margin-bottom:0}.md-typeset table:not([class]) td:not([align]),.md-typeset table:not([class]) th:not([align]){text-align:left}[dir=rtl] .md-typeset table:not([class]) td:not([align]),[dir=rtl] .md-typeset table:not([class]) th:not([align]){text-align:right}.md-typeset table:not([class]) th{font-weight:700;min-width:5rem;padding:.9375em 1.25em;vertical-align:top}.md-typeset table:not([class]) td{border-top:.05rem solid var(--md-typeset-table-color);padding:.9375em 1.25em;vertical-align:top}.md-typeset table:not([class]) tbody tr{transition:background-color 125ms}.md-typeset table:not([class]) tbody tr:hover{background-color:var(--md-typeset-table-color--light);box-shadow:0 .05rem 0 var(--md-default-bg-color) inset}.md-typeset table:not([class]) a{word-break:normal}.md-typeset table th[role=columnheader]{cursor:pointer}[dir=ltr] .md-typeset table th[role=columnheader]:after{margin-left:.5em}[dir=rtl] .md-typeset table th[role=columnheader]:after{margin-right:.5em}.md-typeset table th[role=columnheader]:after{content:"";display:inline-block;height:1.2em;-webkit-mask-image:var(--md-typeset-table-sort-icon);mask-image:var(--md-typeset-table-sort-icon);-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;transition:background-color 125ms;vertical-align:text-bottom;width:1.2em}.md-typeset table th[role=columnheader]:hover:after{background-color:var(--md-default-fg-color--lighter)}.md-typeset table th[role=columnheader][aria-sort=ascending]:after{background-color:var(--md-default-fg-color--light);-webkit-mask-image:var(--md-typeset-table-sort-icon--asc);mask-image:var(--md-typeset-table-sort-icon--asc)}.md-typeset table th[role=columnheader][aria-sort=descending]:after{background-color:var(--md-default-fg-color--light);-webkit-mask-image:var(--md-typeset-table-sort-icon--desc);mask-image:var(--md-typeset-table-sort-icon--desc)}.md-typeset__scrollwrap{margin:1em -.8rem;overflow-x:auto;touch-action:auto}.md-typeset__table{display:inline-block;margin-bottom:.5em;padding:0 .8rem}@media print{.md-typeset__table{display:block}}html .md-typeset__table table{display:table;margin:0;overflow:hidden;width:100%}@media screen and (max-width:44.984375em){.md-content__inner>pre{margin:1em -.8rem}.md-content__inner>pre code{border-radius:0}}.md-typeset .md-author{border-radius:100%;display:block;flex-shrink:0;height:1.6rem;overflow:hidden;position:relative;transition:color 125ms,transform 125ms;width:1.6rem}.md-typeset .md-author img{display:block}.md-typeset .md-author--more{background:var(--md-default-fg-color--lightest);color:var(--md-default-fg-color--lighter);font-size:.6rem;font-weight:700;line-height:1.6rem;text-align:center}.md-typeset .md-author--long{height:2.4rem;width:2.4rem}.md-typeset a.md-author{transform:scale(1)}.md-typeset a.md-author img{filter:grayscale(100%) opacity(75%);transition:filter 125ms}.md-typeset a.md-author:focus,.md-typeset a.md-author:hover{transform:scale(1.1);z-index:1}.md-typeset a.md-author:focus img,.md-typeset a.md-author:hover img{filter:grayscale(0)}.md-banner{background-color:var(--md-footer-bg-color);color:var(--md-footer-fg-color);overflow:auto}@media print{.md-banner{display:none}}.md-banner--warning{background-color:var(--md-warning-bg-color);color:var(--md-warning-fg-color)}.md-banner__inner{font-size:.7rem;margin:.6rem auto;padding:0 .8rem}[dir=ltr] .md-banner__button{float:right}[dir=rtl] .md-banner__button{float:left}.md-banner__button{color:inherit;cursor:pointer;transition:opacity .25s}.no-js .md-banner__button{display:none}.md-banner__button:hover{opacity:.7}html{font-size:125%;height:100%;overflow-x:hidden}@media screen and (min-width:100em){html{font-size:137.5%}}@media screen and (min-width:125em){html{font-size:150%}}body{background-color:var(--md-default-bg-color);display:flex;flex-direction:column;font-size:.5rem;min-height:100%;position:relative;width:100%}@media print{body{display:block}}@media screen and (max-width:59.984375em){body[data-md-scrolllock]{position:fixed}}.md-grid{margin-left:auto;margin-right:auto;max-width:61rem}.md-container{display:flex;flex-direction:column;flex-grow:1}@media print{.md-container{display:block}}.md-main{flex-grow:1}.md-main__inner{display:flex;height:100%;margin-top:1.5rem}.md-ellipsis{overflow:hidden;text-overflow:ellipsis}.md-toggle{display:none}.md-option{height:0;opacity:0;position:absolute;width:0}.md-option:checked+label:not([hidden]){display:block}.md-option.focus-visible+label{outline-color:var(--md-accent-fg-color);outline-style:auto}.md-skip{background-color:var(--md-default-fg-color);border-radius:.1rem;color:var(--md-default-bg-color);font-size:.64rem;margin:.5rem;opacity:0;outline-color:var(--md-accent-fg-color);padding:.3rem .5rem;position:fixed;transform:translateY(.4rem);z-index:-1}.md-skip:focus{opacity:1;transform:translateY(0);transition:transform .25s cubic-bezier(.4,0,.2,1),opacity 175ms 75ms;z-index:10}@page{margin:25mm}:root{--md-clipboard-icon:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 21H8V7h11m0-2H8a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h11a2 2 0 0 0 2-2V7a2 2 0 0 0-2-2m-3-4H4a2 2 0 0 0-2 2v14h2V3h12V1Z"/></svg>')}.md-clipboard{border-radius:.1rem;color:var(--md-default-fg-color--lightest);cursor:pointer;height:1.5em;outline-color:var(--md-accent-fg-color);outline-offset:.1rem;position:absolute;right:.5em;top:.5em;transition:color .25s;width:1.5em;z-index:1}@media print{.md-clipboard{display:none}}.md-clipboard:not(.focus-visible){-webkit-tap-highlight-color:transparent;outline:none}:hover>.md-clipboard{color:var(--md-default-fg-color--light)}.md-clipboard:focus,.md-clipboard:hover{color:var(--md-accent-fg-color)}.md-clipboard:after{background-color:currentcolor;content:"";display:block;height:1.125em;margin:0 auto;-webkit-mask-image:var(--md-clipboard-icon);mask-image:var(--md-clipboard-icon);-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;width:1.125em}.md-clipboard--inline{cursor:pointer}.md-clipboard--inline code{transition:color .25s,background-color .25s}.md-clipboard--inline:focus code,.md-clipboard--inline:hover code{background-color:var(--md-accent-fg-color--transparent);color:var(--md-accent-fg-color)}@keyframes consent{0%{opacity:0;transform:translateY(100%)}to{opacity:1;transform:translateY(0)}}@keyframes overlay{0%{opacity:0}to{opacity:1}}.md-consent__overlay{animation:overlay .25s both;-webkit-backdrop-filter:blur(.1rem);backdrop-filter:blur(.1rem);background-color:#0000008a;height:100%;opacity:1;position:fixed;top:0;width:100%;z-index:5}.md-consent__inner{animation:consent .5s cubic-bezier(.1,.7,.1,1) both;background-color:var(--md-default-bg-color);border:0;border-radius:.1rem;bottom:0;box-shadow:0 0 .2rem #0000001a,0 .2rem .4rem #0003;max-height:100%;overflow:auto;padding:0;position:fixed;width:100%;z-index:5}.md-consent__form{padding:.8rem}.md-consent__settings{display:none;margin:1em 0}input:checked+.md-consent__settings{display:block}.md-consent__controls{margin-bottom:.8rem}.md-typeset .md-consent__controls .md-button{display:inline}@media screen and (max-width:44.984375em){.md-typeset .md-consent__controls .md-button{display:block;margin-top:.4rem;text-align:center;width:100%}}.md-consent label{cursor:pointer}.md-content{flex-grow:1;min-width:0}.md-content__inner{margin:0 .8rem 1.2rem;padding-top:.6rem}@media screen and (min-width:76.25em){[dir=ltr] .md-sidebar--primary:not([hidden])~.md-content>.md-content__inner{margin-left:1.2rem}[dir=ltr] .md-sidebar--secondary:not([hidden])~.md-content>.md-content__inner,[dir=rtl] .md-sidebar--primary:not([hidden])~.md-content>.md-content__inner{margin-right:1.2rem}[dir=rtl] .md-sidebar--secondary:not([hidden])~.md-content>.md-content__inner{margin-left:1.2rem}}.md-content__inner:before{content:"";display:block;height:.4rem}.md-content__inner>:last-child{margin-bottom:0}[dir=ltr] .md-content__button{float:right}[dir=rtl] .md-content__button{float:left}[dir=ltr] .md-content__button{margin-left:.4rem}[dir=rtl] .md-content__button{margin-right:.4rem}.md-content__button{margin:.4rem 0;padding:0}@media print{.md-content__button{display:none}}.md-typeset .md-content__button{color:var(--md-default-fg-color--lighter)}.md-content__button svg{display:inline;vertical-align:top}[dir=rtl] .md-content__button svg{transform:scaleX(-1)}[dir=ltr] .md-dialog{right:.8rem}[dir=rtl] .md-dialog{left:.8rem}.md-dialog{background-color:var(--md-default-fg-color);border-radius:.1rem;bottom:.8rem;box-shadow:var(--md-shadow-z3);min-width:11.1rem;opacity:0;padding:.4rem .6rem;pointer-events:none;position:fixed;transform:translateY(100%);transition:transform 0ms .4s,opacity .4s;z-index:4}@media print{.md-dialog{display:none}}.md-dialog--active{opacity:1;pointer-events:auto;transform:translateY(0);transition:transform .4s cubic-bezier(.075,.85,.175,1),opacity .4s}.md-dialog__inner{color:var(--md-default-bg-color);font-size:.7rem}.md-feedback{margin:2em 0 1em;text-align:center}.md-feedback fieldset{border:none;margin:0;padding:0}.md-feedback__title{font-weight:700;margin:1em auto}.md-feedback__inner{position:relative}.md-feedback__list{display:flex;flex-wrap:wrap;place-content:baseline center;position:relative}.md-feedback__list:hover .md-icon:not(:disabled){color:var(--md-default-fg-color--lighter)}:disabled .md-feedback__list{min-height:1.8rem}.md-feedback__icon{color:var(--md-default-fg-color--light);cursor:pointer;flex-shrink:0;margin:0 .1rem;transition:color 125ms}.md-feedback__icon:not(:disabled).md-icon:hover{color:var(--md-accent-fg-color)}.md-feedback__icon:disabled{color:var(--md-default-fg-color--lightest);pointer-events:none}.md-feedback__note{opacity:0;position:relative;transform:translateY(.4rem);transition:transform .4s cubic-bezier(.1,.7,.1,1),opacity .15s}.md-feedback__note>*{margin:0 auto;max-width:16rem}:disabled .md-feedback__note{opacity:1;transform:translateY(0)}.md-footer{background-color:var(--md-footer-bg-color);color:var(--md-footer-fg-color)}@media print{.md-footer{display:none}}.md-footer__inner{justify-content:space-between;overflow:auto;padding:.2rem}.md-footer__inner:not([hidden]){display:flex}.md-footer__link{align-items:end;display:flex;flex-grow:0.01;margin-bottom:.4rem;margin-top:1rem;max-width:100%;outline-color:var(--md-accent-fg-color);overflow:hidden;transition:opacity .25s}.md-footer__link:focus,.md-footer__link:hover{opacity:.7}[dir=rtl] .md-footer__link svg{transform:scaleX(-1)}@media screen and (max-width:44.984375em){.md-footer__link--prev{flex-shrink:0}.md-footer__link--prev .md-footer__title{display:none}}[dir=ltr] .md-footer__link--next{margin-left:auto}[dir=rtl] .md-footer__link--next{margin-right:auto}.md-footer__link--next{text-align:right}[dir=rtl] .md-footer__link--next{text-align:left}.md-footer__title{flex-grow:1;font-size:.9rem;margin-bottom:.7rem;max-width:calc(100% - 2.4rem);padding:0 1rem;white-space:nowrap}.md-footer__button{margin:.2rem;padding:.4rem}.md-footer__direction{font-size:.64rem;opacity:.7}.md-footer-meta{background-color:var(--md-footer-bg-color--dark)}.md-footer-meta__inner{display:flex;flex-wrap:wrap;justify-content:space-between;padding:.2rem}html .md-footer-meta.md-typeset a{color:var(--md-footer-fg-color--light)}html .md-footer-meta.md-typeset a:focus,html .md-footer-meta.md-typeset a:hover{color:var(--md-footer-fg-color)}.md-copyright{color:var(--md-footer-fg-color--lighter);font-size:.64rem;margin:auto .6rem;padding:.4rem 0;width:100%}@media screen and (min-width:45em){.md-copyright{width:auto}}.md-copyright__highlight{color:var(--md-footer-fg-color--light)}.md-social{display:inline-flex;gap:.2rem;margin:0 .4rem;padding:.2rem 0 .6rem}@media screen and (min-width:45em){.md-social{padding:.6rem 0}}.md-social__link{display:inline-block;height:1.6rem;text-align:center;width:1.6rem}.md-social__link:before{line-height:1.9}.md-social__link svg{fill:currentcolor;max-height:.8rem;vertical-align:-25%}.md-typeset .md-button{border:.1rem solid;border-radius:.1rem;color:var(--md-primary-fg-color);cursor:pointer;display:inline-block;font-weight:700;padding:.625em 2em;transition:color 125ms,background-color 125ms,border-color 125ms}.md-typeset .md-button--primary{background-color:var(--md-primary-fg-color);border-color:var(--md-primary-fg-color);color:var(--md-primary-bg-color)}.md-typeset .md-button:focus,.md-typeset .md-button:hover{background-color:var(--md-accent-fg-color);border-color:var(--md-accent-fg-color);color:var(--md-accent-bg-color)}[dir=ltr] .md-typeset .md-input{border-top-left-radius:.1rem}[dir=ltr] .md-typeset .md-input,[dir=rtl] .md-typeset .md-input{border-top-right-radius:.1rem}[dir=rtl] .md-typeset .md-input{border-top-left-radius:.1rem}.md-typeset .md-input{border-bottom:.1rem solid var(--md-default-fg-color--lighter);box-shadow:var(--md-shadow-z1);font-size:.8rem;height:1.8rem;padding:0 .6rem;transition:border .25s,box-shadow .25s}.md-typeset .md-input:focus,.md-typeset .md-input:hover{border-bottom-color:var(--md-accent-fg-color);box-shadow:var(--md-shadow-z2)}.md-typeset .md-input--stretch{width:100%}.md-header{background-color:var(--md-primary-fg-color);box-shadow:0 0 .2rem #0000,0 .2rem .4rem #0000;color:var(--md-primary-bg-color);display:block;left:0;position:sticky;right:0;top:0;z-index:4}@media print{.md-header{display:none}}.md-header[hidden]{transform:translateY(-100%);transition:transform .25s cubic-bezier(.8,0,.6,1),box-shadow .25s}.md-header--shadow{box-shadow:0 0 .2rem #0000001a,0 .2rem .4rem #0003;transition:transform .25s cubic-bezier(.1,.7,.1,1),box-shadow .25s}.md-header__inner{align-items:center;display:flex;padding:0 .2rem}.md-header__button{color:currentcolor;cursor:pointer;margin:.2rem;outline-color:var(--md-accent-fg-color);padding:.4rem;position:relative;transition:opacity .25s;vertical-align:middle;z-index:1}.md-header__button:hover{opacity:.7}.md-header__button:not([hidden]){display:inline-block}.md-header__button:not(.focus-visible){-webkit-tap-highlight-color:transparent;outline:none}.md-header__button.md-logo{margin:.2rem;padding:.4rem}@media screen and (max-width:76.234375em){.md-header__button.md-logo{display:none}}.md-header__button.md-logo img,.md-header__button.md-logo svg{fill:currentcolor;display:block;height:1.2rem;width:auto}@media screen and (min-width:60em){.md-header__button[for=__search]{display:none}}.no-js .md-header__button[for=__search]{display:none}[dir=rtl] .md-header__button[for=__search] svg{transform:scaleX(-1)}@media screen and (min-width:76.25em){.md-header__button[for=__drawer]{display:none}}.md-header__topic{display:flex;max-width:100%;position:absolute;transition:transform .4s cubic-bezier(.1,.7,.1,1),opacity .15s;white-space:nowrap}.md-header__topic+.md-header__topic{opacity:0;pointer-events:none;transform:translateX(1.25rem);transition:transform .4s cubic-bezier(1,.7,.1,.1),opacity .15s;z-index:-1}[dir=rtl] .md-header__topic+.md-header__topic{transform:translateX(-1.25rem)}.md-header__topic:first-child{font-weight:700}[dir=ltr] .md-header__title{margin-left:1rem;margin-right:.4rem}[dir=rtl] .md-header__title{margin-left:.4rem;margin-right:1rem}.md-header__title{flex-grow:1;font-size:.9rem;height:2.4rem;line-height:2.4rem}.md-header__title--active .md-header__topic{opacity:0;pointer-events:none;transform:translateX(-1.25rem);transition:transform .4s cubic-bezier(1,.7,.1,.1),opacity .15s;z-index:-1}[dir=rtl] .md-header__title--active .md-header__topic{transform:translateX(1.25rem)}.md-header__title--active .md-header__topic+.md-header__topic{opacity:1;pointer-events:auto;transform:translateX(0);transition:transform .4s cubic-bezier(.1,.7,.1,1),opacity .15s;z-index:0}.md-header__title>.md-header__ellipsis{height:100%;position:relative;width:100%}.md-header__option{display:flex;flex-shrink:0;max-width:100%;transition:max-width 0ms .25s,opacity .25s .25s;white-space:nowrap}[data-md-toggle=search]:checked~.md-header .md-header__option{max-width:0;opacity:0;transition:max-width 0ms,opacity 0ms}.md-header__option>input{bottom:0}.md-header__source{display:none}@media screen and (min-width:60em){[dir=ltr] .md-header__source{margin-left:1rem}[dir=rtl] .md-header__source{margin-right:1rem}.md-header__source{display:block;max-width:11.7rem;width:11.7rem}}@media screen and (min-width:76.25em){[dir=ltr] .md-header__source{margin-left:1.4rem}[dir=rtl] .md-header__source{margin-right:1.4rem}}.md-meta{color:var(--md-default-fg-color--light);font-size:.7rem;line-height:1.3}.md-meta__list{display:inline-flex;flex-wrap:wrap;list-style:none;margin:0;padding:0}.md-meta__item:not(:last-child):after{content:"·";margin-left:.2rem;margin-right:.2rem}.md-meta__link{color:var(--md-typeset-a-color)}.md-meta__link:focus,.md-meta__link:hover{color:var(--md-accent-fg-color)}.md-draft{background-color:#ff1744;border-radius:.125em;color:#fff;display:inline-block;font-weight:700;padding-left:.5714285714em;padding-right:.5714285714em}:root{--md-nav-icon--prev:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>');--md-nav-icon--next:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M8.59 16.58 13.17 12 8.59 7.41 10 6l6 6-6 6-1.41-1.42Z"/></svg>');--md-toc-icon:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 9h14V7H3v2m0 4h14v-2H3v2m0 4h14v-2H3v2m16 0h2v-2h-2v2m0-10v2h2V7h-2m0 6h2v-2h-2v2Z"/></svg>')}.md-nav{font-size:.7rem;line-height:1.3}.md-nav__title{color:var(--md-default-fg-color--light);display:block;font-weight:700;overflow:hidden;padding:0 .6rem;text-overflow:ellipsis}.md-nav__title .md-nav__button{display:none}.md-nav__title .md-nav__button img{height:100%;width:auto}.md-nav__title .md-nav__button.md-logo img,.md-nav__title .md-nav__button.md-logo svg{fill:currentcolor;display:block;height:2.4rem;max-width:100%;object-fit:contain;width:auto}.md-nav__list{list-style:none;margin:0;padding:0}.md-nav__link{align-items:flex-start;display:flex;gap:.4rem;margin-top:.625em;scroll-snap-align:start;transition:color 125ms}.md-nav__link--passed{color:var(--md-default-fg-color--light)}.md-nav__item .md-nav__link--active,.md-nav__item .md-nav__link--active code{color:var(--md-typeset-a-color)}.md-nav__link .md-ellipsis{position:relative}[dir=ltr] .md-nav__link .md-icon:last-child{margin-left:auto}[dir=rtl] .md-nav__link .md-icon:last-child{margin-right:auto}.md-nav__link svg{fill:currentcolor;flex-shrink:0;height:1.3em}.md-nav__link[for]:focus,.md-nav__link[for]:hover,.md-nav__link[href]:focus,.md-nav__link[href]:hover{color:var(--md-accent-fg-color);cursor:pointer}.md-nav__link.focus-visible{outline-color:var(--md-accent-fg-color);outline-offset:.2rem}.md-nav--primary .md-nav__link[for=__toc]{display:none}.md-nav--primary .md-nav__link[for=__toc] .md-icon:after{background-color:currentcolor;display:block;height:100%;-webkit-mask-image:var(--md-toc-icon);mask-image:var(--md-toc-icon);width:100%}.md-nav--primary .md-nav__link[for=__toc]~.md-nav{display:none}.md-nav__container>.md-nav__link{margin-top:0}.md-nav__container>.md-nav__link:first-child{flex-grow:1;min-width:0}.md-nav__icon{flex-shrink:0}.md-nav__source{display:none}@media screen and (max-width:76.234375em){.md-nav--primary,.md-nav--primary .md-nav{background-color:var(--md-default-bg-color);display:flex;flex-direction:column;height:100%;left:0;position:absolute;right:0;top:0;z-index:1}.md-nav--primary .md-nav__item,.md-nav--primary .md-nav__title{font-size:.8rem;line-height:1.5}.md-nav--primary .md-nav__title{background-color:var(--md-default-fg-color--lightest);color:var(--md-default-fg-color--light);cursor:pointer;height:5.6rem;line-height:2.4rem;padding:3rem .8rem .2rem;position:relative;white-space:nowrap}[dir=ltr] .md-nav--primary .md-nav__title .md-nav__icon{left:.4rem}[dir=rtl] .md-nav--primary .md-nav__title .md-nav__icon{right:.4rem}.md-nav--primary .md-nav__title .md-nav__icon{display:block;height:1.2rem;margin:.2rem;position:absolute;top:.4rem;width:1.2rem}.md-nav--primary .md-nav__title .md-nav__icon:after{background-color:currentcolor;content:"";display:block;height:100%;-webkit-mask-image:var(--md-nav-icon--prev);mask-image:var(--md-nav-icon--prev);-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;width:100%}.md-nav--primary .md-nav__title~.md-nav__list{background-color:var(--md-default-bg-color);box-shadow:0 .05rem 0 var(--md-default-fg-color--lightest) inset;overflow-y:auto;scroll-snap-type:y mandatory;touch-action:pan-y}.md-nav--primary .md-nav__title~.md-nav__list>:first-child{border-top:0}.md-nav--primary .md-nav__title[for=__drawer]{background-color:var(--md-primary-fg-color);color:var(--md-primary-bg-color);font-weight:700}.md-nav--primary .md-nav__title .md-logo{display:block;left:.2rem;margin:.2rem;padding:.4rem;position:absolute;right:.2rem;top:.2rem}.md-nav--primary .md-nav__list{flex:1}.md-nav--primary .md-nav__item{border-top:.05rem solid var(--md-default-fg-color--lightest)}.md-nav--primary .md-nav__item--active>.md-nav__link{color:var(--md-typeset-a-color)}.md-nav--primary .md-nav__item--active>.md-nav__link:focus,.md-nav--primary .md-nav__item--active>.md-nav__link:hover{color:var(--md-accent-fg-color)}.md-nav--primary .md-nav__link{margin-top:0;padding:.6rem .8rem}.md-nav--primary .md-nav__link svg{margin-top:.1em}.md-nav--primary .md-nav__link>.md-nav__link{padding:0}[dir=ltr] .md-nav--primary .md-nav__link .md-nav__icon{margin-right:-.2rem}[dir=rtl] .md-nav--primary .md-nav__link .md-nav__icon{margin-left:-.2rem}.md-nav--primary .md-nav__link .md-nav__icon{font-size:1.2rem;height:1.2rem;width:1.2rem}.md-nav--primary .md-nav__link .md-nav__icon:after{background-color:currentcolor;content:"";display:block;height:100%;-webkit-mask-image:var(--md-nav-icon--next);mask-image:var(--md-nav-icon--next);-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;width:100%}[dir=rtl] .md-nav--primary .md-nav__icon:after{transform:scale(-1)}.md-nav--primary .md-nav--secondary .md-nav{background-color:initial;position:static}[dir=ltr] .md-nav--primary .md-nav--secondary .md-nav .md-nav__link{padding-left:1.4rem}[dir=rtl] .md-nav--primary .md-nav--secondary .md-nav .md-nav__link{padding-right:1.4rem}[dir=ltr] .md-nav--primary .md-nav--secondary .md-nav .md-nav .md-nav__link{padding-left:2rem}[dir=rtl] .md-nav--primary .md-nav--secondary .md-nav .md-nav .md-nav__link{padding-right:2rem}[dir=ltr] .md-nav--primary .md-nav--secondary .md-nav .md-nav .md-nav .md-nav__link{padding-left:2.6rem}[dir=rtl] .md-nav--primary .md-nav--secondary .md-nav .md-nav .md-nav .md-nav__link{padding-right:2.6rem}[dir=ltr] .md-nav--primary .md-nav--secondary .md-nav .md-nav .md-nav .md-nav .md-nav__link{padding-left:3.2rem}[dir=rtl] .md-nav--primary .md-nav--secondary .md-nav .md-nav .md-nav .md-nav .md-nav__link{padding-right:3.2rem}.md-nav--secondary{background-color:initial}.md-nav__toggle~.md-nav{display:flex;opacity:0;transform:translateX(100%);transition:transform .25s cubic-bezier(.8,0,.6,1),opacity 125ms 50ms}[dir=rtl] .md-nav__toggle~.md-nav{transform:translateX(-100%)}.md-nav__toggle:checked~.md-nav{opacity:1;transform:translateX(0);transition:transform .25s cubic-bezier(.4,0,.2,1),opacity 125ms 125ms}.md-nav__toggle:checked~.md-nav>.md-nav__list{-webkit-backface-visibility:hidden;backface-visibility:hidden}}@media screen and (max-width:59.984375em){.md-nav--primary .md-nav__link[for=__toc]{display:flex}.md-nav--primary .md-nav__link[for=__toc] .md-icon:after{content:""}.md-nav--primary .md-nav__link[for=__toc]+.md-nav__link{display:none}.md-nav--primary .md-nav__link[for=__toc]~.md-nav{display:flex}.md-nav__source{background-color:var(--md-primary-fg-color--dark);color:var(--md-primary-bg-color);display:block;padding:0 .2rem}}@media screen and (min-width:60em) and (max-width:76.234375em){.md-nav--integrated .md-nav__link[for=__toc]{display:flex}.md-nav--integrated .md-nav__link[for=__toc] .md-icon:after{content:""}.md-nav--integrated .md-nav__link[for=__toc]+.md-nav__link{display:none}.md-nav--integrated .md-nav__link[for=__toc]~.md-nav{display:flex}}@media screen and (min-width:60em){.md-nav{margin-bottom:-.4rem}.md-nav--secondary .md-nav__title{background:var(--md-default-bg-color);box-shadow:0 0 .4rem .4rem var(--md-default-bg-color);position:sticky;top:0;z-index:1}.md-nav--secondary .md-nav__title[for=__toc]{scroll-snap-align:start}.md-nav--secondary .md-nav__title .md-nav__icon{display:none}[dir=ltr] .md-nav--secondary .md-nav__list{padding-left:.6rem}[dir=rtl] .md-nav--secondary .md-nav__list{padding-right:.6rem}.md-nav--secondary .md-nav__list{padding-bottom:.4rem}[dir=ltr] .md-nav--secondary .md-nav__item>.md-nav__link{margin-right:.4rem}[dir=rtl] .md-nav--secondary .md-nav__item>.md-nav__link{margin-left:.4rem}}@media screen and (min-width:76.25em){.md-nav{margin-bottom:-.4rem;transition:max-height .25s cubic-bezier(.86,0,.07,1)}.md-nav--primary .md-nav__title{background:var(--md-default-bg-color);box-shadow:0 0 .4rem .4rem var(--md-default-bg-color);position:sticky;top:0;z-index:1}.md-nav--primary .md-nav__title[for=__drawer]{scroll-snap-align:start}.md-nav--primary .md-nav__title .md-nav__icon{display:none}[dir=ltr] .md-nav--primary .md-nav__list{padding-left:.6rem}[dir=rtl] .md-nav--primary .md-nav__list{padding-right:.6rem}.md-nav--primary .md-nav__list{padding-bottom:.4rem}[dir=ltr] .md-nav--primary .md-nav__item>.md-nav__link{margin-right:.4rem}[dir=rtl] .md-nav--primary .md-nav__item>.md-nav__link{margin-left:.4rem}.md-nav__toggle~.md-nav{display:grid;grid-template-rows:0fr;opacity:0;transition:grid-template-rows .25s cubic-bezier(.86,0,.07,1),opacity .25s,visibility 0ms .25s;visibility:collapse}.md-nav__toggle~.md-nav>.md-nav__list{overflow:hidden}.md-nav__toggle.md-toggle--indeterminate~.md-nav,.md-nav__toggle:checked~.md-nav{grid-template-rows:1fr;opacity:1;transition:grid-template-rows .25s cubic-bezier(.86,0,.07,1),opacity .15s .1s,visibility 0ms;visibility:visible}.md-nav__toggle.md-toggle--indeterminate~.md-nav{transition:none}.md-nav__item--nested>.md-nav>.md-nav__title{display:none}.md-nav__item--section{display:block;margin:1.25em 0}.md-nav__item--section:last-child{margin-bottom:0}.md-nav__item--section>.md-nav__link{font-weight:700}.md-nav__item--section>.md-nav__link[for]{color:var(--md-default-fg-color--light)}.md-nav__item--section>.md-nav__link:not(.md-nav__container){pointer-events:none}.md-nav__item--section>.md-nav__link .md-icon,.md-nav__item--section>.md-nav__link>[for]{display:none}[dir=ltr] .md-nav__item--section>.md-nav{margin-left:-.6rem}[dir=rtl] .md-nav__item--section>.md-nav{margin-right:-.6rem}.md-nav__item--section>.md-nav{display:block;opacity:1;visibility:visible}.md-nav__item--section>.md-nav>.md-nav__list>.md-nav__item{padding:0}.md-nav__icon{border-radius:100%;height:.9rem;transition:background-color .25s;width:.9rem}.md-nav__icon:hover{background-color:var(--md-accent-fg-color--transparent)}.md-nav__icon:after{background-color:currentcolor;border-radius:100%;content:"";display:inline-block;height:100%;-webkit-mask-image:var(--md-nav-icon--next);mask-image:var(--md-nav-icon--next);-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;transition:transform .25s;vertical-align:-.1rem;width:100%}[dir=rtl] .md-nav__icon:after{transform:rotate(180deg)}.md-nav__item--nested .md-nav__toggle:checked~.md-nav__link .md-nav__icon:after,.md-nav__item--nested .md-toggle--indeterminate~.md-nav__link .md-nav__icon:after{transform:rotate(90deg)}.md-nav--lifted>.md-nav__list>.md-nav__item,.md-nav--lifted>.md-nav__title{display:none}.md-nav--lifted>.md-nav__list>.md-nav__item--active{display:block}.md-nav--lifted>.md-nav__list>.md-nav__item--active>.md-nav__link{background:var(--md-default-bg-color);box-shadow:0 0 .4rem .4rem var(--md-default-bg-color);margin-top:0;position:sticky;top:0;z-index:1}.md-nav--lifted>.md-nav__list>.md-nav__item--active>.md-nav__link:not(.md-nav__container){pointer-events:none}.md-nav--lifted>.md-nav__list>.md-nav__item--active.md-nav__item--section{margin:0}[dir=ltr] .md-nav--lifted>.md-nav__list>.md-nav__item>.md-nav:not(.md-nav--secondary){margin-left:-.6rem}[dir=rtl] .md-nav--lifted>.md-nav__list>.md-nav__item>.md-nav:not(.md-nav--secondary){margin-right:-.6rem}.md-nav--lifted>.md-nav__list>.md-nav__item>[for]{color:var(--md-default-fg-color--light)}.md-nav--lifted .md-nav[data-md-level="1"]{grid-template-rows:1fr;opacity:1;visibility:visible}[dir=ltr] .md-nav--integrated>.md-nav__list>.md-nav__item--active .md-nav--secondary{border-left:.05rem solid var(--md-primary-fg-color)}[dir=rtl] .md-nav--integrated>.md-nav__list>.md-nav__item--active .md-nav--secondary{border-right:.05rem solid var(--md-primary-fg-color)}.md-nav--integrated>.md-nav__list>.md-nav__item--active .md-nav--secondary{display:block;margin-bottom:1.25em;opacity:1;visibility:visible}.md-nav--integrated>.md-nav__list>.md-nav__item--active .md-nav--secondary>.md-nav__list{overflow:visible;padding-bottom:0}.md-nav--integrated>.md-nav__list>.md-nav__item--active .md-nav--secondary>.md-nav__title{display:none}}.md-pagination{font-size:.8rem;font-weight:700;gap:.4rem}.md-pagination,.md-pagination>*{align-items:center;display:flex;justify-content:center}.md-pagination>*{border-radius:.2rem;height:1.8rem;min-width:1.8rem;text-align:center}.md-pagination__current{background-color:var(--md-default-fg-color--lightest);color:var(--md-default-fg-color--light)}.md-pagination__link{transition:color 125ms,background-color 125ms}.md-pagination__link:focus,.md-pagination__link:hover{background-color:var(--md-accent-fg-color--transparent);color:var(--md-accent-fg-color)}.md-pagination__link:focus svg,.md-pagination__link:hover svg{color:var(--md-accent-fg-color)}.md-pagination__link.focus-visible{outline-color:var(--md-accent-fg-color);outline-offset:.2rem}.md-pagination__link svg{fill:currentcolor;color:var(--md-default-fg-color--lighter);display:block;max-height:100%;width:1.2rem}.md-post__back{border-bottom:.05rem solid var(--md-default-fg-color--lightest);margin-bottom:1.2rem;padding-bottom:1.2rem}@media screen and (max-width:76.234375em){.md-post__back{display:none}}[dir=rtl] .md-post__back svg{transform:scaleX(-1)}.md-post__authors{display:flex;flex-direction:column;gap:.6rem;margin:0 .6rem 1.2rem}.md-post .md-post__meta a{transition:color 125ms}.md-post .md-post__meta a:focus,.md-post .md-post__meta a:hover{color:var(--md-accent-fg-color)}.md-post__title{color:var(--md-default-fg-color--light);font-weight:700}.md-post--excerpt{margin-bottom:3.2rem}.md-post--excerpt .md-post__header{align-items:center;display:flex;gap:.6rem;min-height:1.6rem}.md-post--excerpt .md-post__authors{align-items:center;display:inline-flex;flex-direction:row;gap:.2rem;margin:0;min-height:2.4rem}[dir=ltr] .md-post--excerpt .md-post__meta .md-meta__list{margin-right:.4rem}[dir=rtl] .md-post--excerpt .md-post__meta .md-meta__list{margin-left:.4rem}.md-post--excerpt .md-post__content>:first-child{--md-scroll-margin:6rem;margin-top:0}.md-post>.md-nav--secondary{margin:1em 0}.md-profile{align-items:center;display:flex;font-size:.7rem;gap:.6rem;line-height:1.4;width:100%}.md-profile__description{flex-grow:1}.md-content--post{display:flex}@media screen and (max-width:76.234375em){.md-content--post{flex-flow:column-reverse}}.md-content--post>.md-content__inner{min-width:0}@media screen and (min-width:76.25em){[dir=ltr] .md-content--post>.md-content__inner{margin-left:1.2rem}[dir=rtl] .md-content--post>.md-content__inner{margin-right:1.2rem}}@media screen and (max-width:76.234375em){.md-sidebar.md-sidebar--post{padding:0;position:static;width:100%}.md-sidebar.md-sidebar--post .md-sidebar__scrollwrap{overflow:visible}.md-sidebar.md-sidebar--post .md-sidebar__inner{padding:0}.md-sidebar.md-sidebar--post .md-post__meta{margin-left:.6rem;margin-right:.6rem}.md-sidebar.md-sidebar--post .md-nav__item{border:none;display:inline}.md-sidebar.md-sidebar--post .md-nav__list{display:inline-flex;flex-wrap:wrap;gap:.6rem;padding-bottom:.6rem;padding-top:.6rem}.md-sidebar.md-sidebar--post .md-nav__link{padding:0}.md-sidebar.md-sidebar--post .md-nav{height:auto;margin-bottom:0;position:static}}:root{--md-progress-value:0;--md-progress-delay:400ms}.md-progress{background:var(--md-primary-bg-color);height:.075rem;opacity:min(clamp(0,var(--md-progress-value),1),clamp(0,100 - var(--md-progress-value),1));position:fixed;top:0;transform:scaleX(calc(var(--md-progress-value)*1%));transform-origin:left;transition:transform .5s cubic-bezier(.19,1,.22,1),opacity .25s var(--md-progress-delay);width:100%;z-index:4}:root{--md-search-result-icon:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h7c-.41-.25-.8-.56-1.14-.9-.33-.33-.61-.7-.86-1.1H6V4h7v5h5v1.18c.71.16 1.39.43 2 .82V8l-6-6m6.31 16.9c1.33-2.11.69-4.9-1.4-6.22-2.11-1.33-4.91-.68-6.22 1.4-1.34 2.11-.69 4.89 1.4 6.22 1.46.93 3.32.93 4.79.02L22 23.39 23.39 22l-3.08-3.1m-3.81.1a2.5 2.5 0 0 1-2.5-2.5 2.5 2.5 0 0 1 2.5-2.5 2.5 2.5 0 0 1 2.5 2.5 2.5 2.5 0 0 1-2.5 2.5Z"/></svg>')}.md-search{position:relative}@media screen and (min-width:60em){.md-search{padding:.2rem 0}}.no-js .md-search{display:none}.md-search__overlay{opacity:0;z-index:1}@media screen and (max-width:59.984375em){[dir=ltr] .md-search__overlay{left:-2.2rem}[dir=rtl] .md-search__overlay{right:-2.2rem}.md-search__overlay{background-color:var(--md-default-bg-color);border-radius:1rem;height:2rem;overflow:hidden;pointer-events:none;position:absolute;top:-1rem;transform-origin:center;transition:transform .3s .1s,opacity .2s .2s;width:2rem}[data-md-toggle=search]:checked~.md-header .md-search__overlay{opacity:1;transition:transform .4s,opacity .1s}}@media screen and (min-width:60em){[dir=ltr] .md-search__overlay{left:0}[dir=rtl] .md-search__overlay{right:0}.md-search__overlay{background-color:#0000008a;cursor:pointer;height:0;position:fixed;top:0;transition:width 0ms .25s,height 0ms .25s,opacity .25s;width:0}[data-md-toggle=search]:checked~.md-header .md-search__overlay{height:200vh;opacity:1;transition:width 0ms,height 0ms,opacity .25s;width:100%}}@media screen and (max-width:29.984375em){[data-md-toggle=search]:checked~.md-header .md-search__overlay{transform:scale(45)}}@media screen and (min-width:30em) and (max-width:44.984375em){[data-md-toggle=search]:checked~.md-header .md-search__overlay{transform:scale(60)}}@media screen and (min-width:45em) and (max-width:59.984375em){[data-md-toggle=search]:checked~.md-header .md-search__overlay{transform:scale(75)}}.md-search__inner{-webkit-backface-visibility:hidden;backface-visibility:hidden}@media screen and (max-width:59.984375em){[dir=ltr] .md-search__inner{left:0}[dir=rtl] .md-search__inner{right:0}.md-search__inner{height:0;opacity:0;overflow:hidden;position:fixed;top:0;transform:translateX(5%);transition:width 0ms .3s,height 0ms .3s,transform .15s cubic-bezier(.4,0,.2,1) .15s,opacity .15s .15s;width:0;z-index:2}[dir=rtl] .md-search__inner{transform:translateX(-5%)}[data-md-toggle=search]:checked~.md-header .md-search__inner{height:100%;opacity:1;transform:translateX(0);transition:width 0ms 0ms,height 0ms 0ms,transform .15s cubic-bezier(.1,.7,.1,1) .15s,opacity .15s .15s;width:100%}}@media screen and (min-width:60em){[dir=ltr] .md-search__inner{float:right}[dir=rtl] .md-search__inner{float:left}.md-search__inner{padding:.1rem 0;position:relative;transition:width .25s cubic-bezier(.1,.7,.1,1);width:11.7rem}}@media screen and (min-width:60em) and (max-width:76.234375em){[data-md-toggle=search]:checked~.md-header .md-search__inner{width:23.4rem}}@media screen and (min-width:76.25em){[data-md-toggle=search]:checked~.md-header .md-search__inner{width:34.4rem}}.md-search__form{background-color:var(--md-default-bg-color);box-shadow:0 0 .6rem #0000;height:2.4rem;position:relative;transition:color .25s,background-color .25s;z-index:2}@media screen and (min-width:60em){.md-search__form{background-color:#00000042;border-radius:.1rem;height:1.8rem}.md-search__form:hover{background-color:#ffffff1f}}[data-md-toggle=search]:checked~.md-header .md-search__form{background-color:var(--md-default-bg-color);border-radius:.1rem .1rem 0 0;box-shadow:0 0 .6rem #00000012;color:var(--md-default-fg-color)}[dir=ltr] .md-search__input{padding-left:3.6rem;padding-right:2.2rem}[dir=rtl] .md-search__input{padding-left:2.2rem;padding-right:3.6rem}.md-search__input{background:#0000;font-size:.9rem;height:100%;position:relative;text-overflow:ellipsis;width:100%;z-index:2}.md-search__input::placeholder{transition:color .25s}.md-search__input::placeholder,.md-search__input~.md-search__icon{color:var(--md-default-fg-color--light)}.md-search__input::-ms-clear{display:none}@media screen and (max-width:59.984375em){.md-search__input{font-size:.9rem;height:2.4rem;width:100%}}@media screen and (min-width:60em){[dir=ltr] .md-search__input{padding-left:2.2rem}[dir=rtl] .md-search__input{padding-right:2.2rem}.md-search__input{color:inherit;font-size:.8rem}.md-search__input::placeholder{color:var(--md-primary-bg-color--light)}.md-search__input+.md-search__icon{color:var(--md-primary-bg-color)}[data-md-toggle=search]:checked~.md-header .md-search__input{text-overflow:clip}[data-md-toggle=search]:checked~.md-header .md-search__input+.md-search__icon{color:var(--md-default-fg-color--light)}[data-md-toggle=search]:checked~.md-header .md-search__input::placeholder{color:#0000}}.md-search__icon{cursor:pointer;display:inline-block;height:1.2rem;transition:color .25s,opacity .25s;width:1.2rem}.md-search__icon:hover{opacity:.7}[dir=ltr] .md-search__icon[for=__search]{left:.5rem}[dir=rtl] .md-search__icon[for=__search]{right:.5rem}.md-search__icon[for=__search]{position:absolute;top:.3rem;z-index:2}[dir=rtl] .md-search__icon[for=__search] svg{transform:scaleX(-1)}@media screen and (max-width:59.984375em){[dir=ltr] .md-search__icon[for=__search]{left:.8rem}[dir=rtl] .md-search__icon[for=__search]{right:.8rem}.md-search__icon[for=__search]{top:.6rem}.md-search__icon[for=__search] svg:first-child{display:none}}@media screen and (min-width:60em){.md-search__icon[for=__search]{pointer-events:none}.md-search__icon[for=__search] svg:last-child{display:none}}[dir=ltr] .md-search__options{right:.5rem}[dir=rtl] .md-search__options{left:.5rem}.md-search__options{pointer-events:none;position:absolute;top:.3rem;z-index:2}@media screen and (max-width:59.984375em){[dir=ltr] .md-search__options{right:.8rem}[dir=rtl] .md-search__options{left:.8rem}.md-search__options{top:.6rem}}[dir=ltr] .md-search__options>.md-icon{margin-left:.2rem}[dir=rtl] .md-search__options>.md-icon{margin-right:.2rem}.md-search__options>.md-icon{color:var(--md-default-fg-color--light);opacity:0;transform:scale(.75);transition:transform .15s cubic-bezier(.1,.7,.1,1),opacity .15s}.md-search__options>.md-icon:not(.focus-visible){-webkit-tap-highlight-color:transparent;outline:none}[data-md-toggle=search]:checked~.md-header .md-search__input:valid~.md-search__options>.md-icon{opacity:1;pointer-events:auto;transform:scale(1)}[data-md-toggle=search]:checked~.md-header .md-search__input:valid~.md-search__options>.md-icon:hover{opacity:.7}[dir=ltr] .md-search__suggest{padding-left:3.6rem;padding-right:2.2rem}[dir=rtl] .md-search__suggest{padding-left:2.2rem;padding-right:3.6rem}.md-search__suggest{align-items:center;color:var(--md-default-fg-color--lighter);display:flex;font-size:.9rem;height:100%;opacity:0;position:absolute;top:0;transition:opacity 50ms;white-space:nowrap;width:100%}@media screen and (min-width:60em){[dir=ltr] .md-search__suggest{padding-left:2.2rem}[dir=rtl] .md-search__suggest{padding-right:2.2rem}.md-search__suggest{font-size:.8rem}}[data-md-toggle=search]:checked~.md-header .md-search__suggest{opacity:1;transition:opacity .3s .1s}[dir=ltr] .md-search__output{border-bottom-left-radius:.1rem}[dir=ltr] .md-search__output,[dir=rtl] .md-search__output{border-bottom-right-radius:.1rem}[dir=rtl] .md-search__output{border-bottom-left-radius:.1rem}.md-search__output{overflow:hidden;position:absolute;width:100%;z-index:1}@media screen and (max-width:59.984375em){.md-search__output{bottom:0;top:2.4rem}}@media screen and (min-width:60em){.md-search__output{opacity:0;top:1.9rem;transition:opacity .4s}[data-md-toggle=search]:checked~.md-header .md-search__output{box-shadow:var(--md-shadow-z3);opacity:1}}.md-search__scrollwrap{-webkit-backface-visibility:hidden;backface-visibility:hidden;background-color:var(--md-default-bg-color);height:100%;overflow-y:auto;touch-action:pan-y}@media (-webkit-max-device-pixel-ratio:1),(max-resolution:1dppx){.md-search__scrollwrap{transform:translateZ(0)}}@media screen and (min-width:60em) and (max-width:76.234375em){.md-search__scrollwrap{width:23.4rem}}@media screen and (min-width:76.25em){.md-search__scrollwrap{width:34.4rem}}@media screen and (min-width:60em){.md-search__scrollwrap{max-height:0;scrollbar-color:var(--md-default-fg-color--lighter) #0000;scrollbar-width:thin}[data-md-toggle=search]:checked~.md-header .md-search__scrollwrap{max-height:75vh}.md-search__scrollwrap:hover{scrollbar-color:var(--md-accent-fg-color) #0000}.md-search__scrollwrap::-webkit-scrollbar{height:.2rem;width:.2rem}.md-search__scrollwrap::-webkit-scrollbar-thumb{background-color:var(--md-default-fg-color--lighter)}.md-search__scrollwrap::-webkit-scrollbar-thumb:hover{background-color:var(--md-accent-fg-color)}}.md-search-result{color:var(--md-default-fg-color);word-break:break-word}.md-search-result__meta{background-color:var(--md-default-fg-color--lightest);color:var(--md-default-fg-color--light);font-size:.64rem;line-height:1.8rem;padding:0 .8rem;scroll-snap-align:start}@media screen and (min-width:60em){[dir=ltr] .md-search-result__meta{padding-left:2.2rem}[dir=rtl] .md-search-result__meta{padding-right:2.2rem}}.md-search-result__list{list-style:none;margin:0;padding:0;-webkit-user-select:none;user-select:none}.md-search-result__item{box-shadow:0 -.05rem var(--md-default-fg-color--lightest)}.md-search-result__item:first-child{box-shadow:none}.md-search-result__link{display:block;outline:none;scroll-snap-align:start;transition:background-color .25s}.md-search-result__link:focus,.md-search-result__link:hover{background-color:var(--md-accent-fg-color--transparent)}.md-search-result__link:last-child p:last-child{margin-bottom:.6rem}.md-search-result__more>summary{cursor:pointer;display:block;outline:none;position:sticky;scroll-snap-align:start;top:0;z-index:1}.md-search-result__more>summary::marker{display:none}.md-search-result__more>summary::-webkit-details-marker{display:none}.md-search-result__more>summary>div{color:var(--md-typeset-a-color);font-size:.64rem;padding:.75em .8rem;transition:color .25s,background-color .25s}@media screen and (min-width:60em){[dir=ltr] .md-search-result__more>summary>div{padding-left:2.2rem}[dir=rtl] .md-search-result__more>summary>div{padding-right:2.2rem}}.md-search-result__more>summary:focus>div,.md-search-result__more>summary:hover>div{background-color:var(--md-accent-fg-color--transparent);color:var(--md-accent-fg-color)}.md-search-result__more[open]>summary{background-color:var(--md-default-bg-color)}.md-search-result__article{overflow:hidden;padding:0 .8rem;position:relative}@media screen and (min-width:60em){[dir=ltr] .md-search-result__article{padding-left:2.2rem}[dir=rtl] .md-search-result__article{padding-right:2.2rem}}[dir=ltr] .md-search-result__icon{left:0}[dir=rtl] .md-search-result__icon{right:0}.md-search-result__icon{color:var(--md-default-fg-color--light);height:1.2rem;margin:.5rem;position:absolute;width:1.2rem}@media screen and (max-width:59.984375em){.md-search-result__icon{display:none}}.md-search-result__icon:after{background-color:currentcolor;content:"";display:inline-block;height:100%;-webkit-mask-image:var(--md-search-result-icon);mask-image:var(--md-search-result-icon);-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;width:100%}[dir=rtl] .md-search-result__icon:after{transform:scaleX(-1)}.md-search-result .md-typeset{color:var(--md-default-fg-color--light);font-size:.64rem;line-height:1.6}.md-search-result .md-typeset h1{color:var(--md-default-fg-color);font-size:.8rem;font-weight:400;line-height:1.4;margin:.55rem 0}.md-search-result .md-typeset h1 mark{text-decoration:none}.md-search-result .md-typeset h2{color:var(--md-default-fg-color);font-size:.64rem;font-weight:700;line-height:1.6;margin:.5em 0}.md-search-result .md-typeset h2 mark{text-decoration:none}.md-search-result__terms{color:var(--md-default-fg-color);display:block;font-size:.64rem;font-style:italic;margin:.5em 0}.md-search-result mark{background-color:initial;color:var(--md-accent-fg-color);text-decoration:underline}.md-select{position:relative;z-index:1}.md-select__inner{background-color:var(--md-default-bg-color);border-radius:.1rem;box-shadow:var(--md-shadow-z2);color:var(--md-default-fg-color);left:50%;margin-top:.2rem;max-height:0;opacity:0;position:absolute;top:calc(100% - .2rem);transform:translate3d(-50%,.3rem,0);transition:transform .25s 375ms,opacity .25s .25s,max-height 0ms .5s}.md-select:focus-within .md-select__inner,.md-select:hover .md-select__inner{max-height:10rem;opacity:1;transform:translate3d(-50%,0,0);transition:transform .25s cubic-bezier(.1,.7,.1,1),opacity .25s,max-height 0ms}.md-select__inner:after{border-bottom:.2rem solid #0000;border-bottom-color:var(--md-default-bg-color);border-left:.2rem solid #0000;border-right:.2rem solid #0000;border-top:0;content:"";height:0;left:50%;margin-left:-.2rem;margin-top:-.2rem;position:absolute;top:0;width:0}.md-select__list{border-radius:.1rem;font-size:.8rem;list-style-type:none;margin:0;max-height:inherit;overflow:auto;padding:0}.md-select__item{line-height:1.8rem}[dir=ltr] .md-select__link{padding-left:.6rem;padding-right:1.2rem}[dir=rtl] .md-select__link{padding-left:1.2rem;padding-right:.6rem}.md-select__link{cursor:pointer;display:block;outline:none;scroll-snap-align:start;transition:background-color .25s,color .25s;width:100%}.md-select__link:focus,.md-select__link:hover{color:var(--md-accent-fg-color)}.md-select__link:focus{background-color:var(--md-default-fg-color--lightest)}.md-sidebar{align-self:flex-start;flex-shrink:0;padding:1.2rem 0;position:sticky;top:2.4rem;width:12.1rem}@media print{.md-sidebar{display:none}}@media screen and (max-width:76.234375em){[dir=ltr] .md-sidebar--primary{left:-12.1rem}[dir=rtl] .md-sidebar--primary{right:-12.1rem}.md-sidebar--primary{background-color:var(--md-default-bg-color);display:block;height:100%;position:fixed;top:0;transform:translateX(0);transition:transform .25s cubic-bezier(.4,0,.2,1),box-shadow .25s;width:12.1rem;z-index:5}[data-md-toggle=drawer]:checked~.md-container .md-sidebar--primary{box-shadow:var(--md-shadow-z3);transform:translateX(12.1rem)}[dir=rtl] [data-md-toggle=drawer]:checked~.md-container .md-sidebar--primary{transform:translateX(-12.1rem)}.md-sidebar--primary .md-sidebar__scrollwrap{bottom:0;left:0;margin:0;overflow:hidden;position:absolute;right:0;scroll-snap-type:none;top:0}}@media screen and (min-width:76.25em){.md-sidebar{height:0}.no-js .md-sidebar{height:auto}.md-header--lifted~.md-container .md-sidebar{top:4.8rem}}.md-sidebar--secondary{display:none;order:2}@media screen and (min-width:60em){.md-sidebar--secondary{height:0}.no-js .md-sidebar--secondary{height:auto}.md-sidebar--secondary:not([hidden]){display:block}.md-sidebar--secondary .md-sidebar__scrollwrap{touch-action:pan-y}}.md-sidebar__scrollwrap{scrollbar-gutter:stable;-webkit-backface-visibility:hidden;backface-visibility:hidden;margin:0 .2rem;overflow-y:auto;scrollbar-color:var(--md-default-fg-color--lighter) #0000;scrollbar-width:thin}.md-sidebar__scrollwrap::-webkit-scrollbar{height:.2rem;width:.2rem}.md-sidebar__scrollwrap:focus-within,.md-sidebar__scrollwrap:hover{scrollbar-color:var(--md-accent-fg-color) #0000}.md-sidebar__scrollwrap:focus-within::-webkit-scrollbar-thumb,.md-sidebar__scrollwrap:hover::-webkit-scrollbar-thumb{background-color:var(--md-default-fg-color--lighter)}.md-sidebar__scrollwrap:focus-within::-webkit-scrollbar-thumb:hover,.md-sidebar__scrollwrap:hover::-webkit-scrollbar-thumb:hover{background-color:var(--md-accent-fg-color)}@supports selector(::-webkit-scrollbar){.md-sidebar__scrollwrap{scrollbar-gutter:auto}[dir=ltr] .md-sidebar__inner{padding-right:calc(100% - 11.5rem)}[dir=rtl] .md-sidebar__inner{padding-left:calc(100% - 11.5rem)}}@media screen and (max-width:76.234375em){.md-overlay{background-color:#0000008a;height:0;opacity:0;position:fixed;top:0;transition:width 0ms .25s,height 0ms .25s,opacity .25s;width:0;z-index:5}[data-md-toggle=drawer]:checked~.md-overlay{height:100%;opacity:1;transition:width 0ms,height 0ms,opacity .25s;width:100%}}@keyframes facts{0%{height:0}to{height:.65rem}}@keyframes fact{0%{opacity:0;transform:translateY(100%)}50%{opacity:0}to{opacity:1;transform:translateY(0)}}:root{--md-source-forks-icon:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path d="M5 5.372v.878c0 .414.336.75.75.75h4.5a.75.75 0 0 0 .75-.75v-.878a2.25 2.25 0 1 1 1.5 0v.878a2.25 2.25 0 0 1-2.25 2.25h-1.5v2.128a2.251 2.251 0 1 1-1.5 0V8.5h-1.5A2.25 2.25 0 0 1 3.5 6.25v-.878a2.25 2.25 0 1 1 1.5 0ZM5 3.25a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Zm6.75.75a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5Zm-3 8.75a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Z"/></svg>');--md-source-repositories-icon:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path d="M2 2.5A2.5 2.5 0 0 1 4.5 0h8.75a.75.75 0 0 1 .75.75v12.5a.75.75 0 0 1-.75.75h-2.5a.75.75 0 0 1 0-1.5h1.75v-2h-8a1 1 0 0 0-.714 1.7.75.75 0 1 1-1.072 1.05A2.495 2.495 0 0 1 2 11.5Zm10.5-1h-8a1 1 0 0 0-1 1v6.708A2.486 2.486 0 0 1 4.5 9h8ZM5 12.25a.25.25 0 0 1 .25-.25h3.5a.25.25 0 0 1 .25.25v3.25a.25.25 0 0 1-.4.2l-1.45-1.087a.249.249 0 0 0-.3 0L5.4 15.7a.25.25 0 0 1-.4-.2Z"/></svg>');--md-source-stars-icon:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path d="M8 .25a.75.75 0 0 1 .673.418l1.882 3.815 4.21.612a.75.75 0 0 1 .416 1.279l-3.046 2.97.719 4.192a.751.751 0 0 1-1.088.791L8 12.347l-3.766 1.98a.75.75 0 0 1-1.088-.79l.72-4.194L.818 6.374a.75.75 0 0 1 .416-1.28l4.21-.611L7.327.668A.75.75 0 0 1 8 .25Zm0 2.445L6.615 5.5a.75.75 0 0 1-.564.41l-3.097.45 2.24 2.184a.75.75 0 0 1 .216.664l-.528 3.084 2.769-1.456a.75.75 0 0 1 .698 0l2.77 1.456-.53-3.084a.75.75 0 0 1 .216-.664l2.24-2.183-3.096-.45a.75.75 0 0 1-.564-.41L8 2.694Z"/></svg>');--md-source-version-icon:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path d="M1 7.775V2.75C1 1.784 1.784 1 2.75 1h5.025c.464 0 .91.184 1.238.513l6.25 6.25a1.75 1.75 0 0 1 0 2.474l-5.026 5.026a1.75 1.75 0 0 1-2.474 0l-6.25-6.25A1.752 1.752 0 0 1 1 7.775Zm1.5 0c0 .066.026.13.073.177l6.25 6.25a.25.25 0 0 0 .354 0l5.025-5.025a.25.25 0 0 0 0-.354l-6.25-6.25a.25.25 0 0 0-.177-.073H2.75a.25.25 0 0 0-.25.25ZM6 5a1 1 0 1 1 0 2 1 1 0 0 1 0-2Z"/></svg>')}.md-source{-webkit-backface-visibility:hidden;backface-visibility:hidden;display:block;font-size:.65rem;line-height:1.2;outline-color:var(--md-accent-fg-color);transition:opacity .25s;white-space:nowrap}.md-source:hover{opacity:.7}.md-source__icon{display:inline-block;height:2.4rem;vertical-align:middle;width:2rem}[dir=ltr] .md-source__icon svg{margin-left:.6rem}[dir=rtl] .md-source__icon svg{margin-right:.6rem}.md-source__icon svg{margin-top:.6rem}[dir=ltr] .md-source__icon+.md-source__repository{padding-left:2rem}[dir=rtl] .md-source__icon+.md-source__repository{padding-right:2rem}[dir=ltr] .md-source__icon+.md-source__repository{margin-left:-2rem}[dir=rtl] .md-source__icon+.md-source__repository{margin-right:-2rem}[dir=ltr] .md-source__repository{margin-left:.6rem}[dir=rtl] .md-source__repository{margin-right:.6rem}.md-source__repository{display:inline-block;max-width:calc(100% - 1.2rem);overflow:hidden;text-overflow:ellipsis;vertical-align:middle}.md-source__facts{display:flex;font-size:.55rem;gap:.4rem;list-style-type:none;margin:.1rem 0 0;opacity:.75;overflow:hidden;padding:0;width:100%}.md-source__repository--active .md-source__facts{animation:facts .25s ease-in}.md-source__fact{overflow:hidden;text-overflow:ellipsis}.md-source__repository--active .md-source__fact{animation:fact .4s ease-out}[dir=ltr] .md-source__fact:before{margin-right:.1rem}[dir=rtl] .md-source__fact:before{margin-left:.1rem}.md-source__fact:before{background-color:currentcolor;content:"";display:inline-block;height:.6rem;-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;vertical-align:text-top;width:.6rem}.md-source__fact:nth-child(1n+2){flex-shrink:0}.md-source__fact--version:before{-webkit-mask-image:var(--md-source-version-icon);mask-image:var(--md-source-version-icon)}.md-source__fact--stars:before{-webkit-mask-image:var(--md-source-stars-icon);mask-image:var(--md-source-stars-icon)}.md-source__fact--forks:before{-webkit-mask-image:var(--md-source-forks-icon);mask-image:var(--md-source-forks-icon)}.md-source__fact--repositories:before{-webkit-mask-image:var(--md-source-repositories-icon);mask-image:var(--md-source-repositories-icon)}.md-source-file{margin:1em 0}[dir=ltr] .md-source-file__fact{margin-right:.6rem}[dir=rtl] .md-source-file__fact{margin-left:.6rem}.md-source-file__fact{align-items:center;color:var(--md-default-fg-color--light);display:inline-flex;font-size:.68rem;gap:.3rem}.md-source-file__fact .md-icon{flex-shrink:0;margin-bottom:.05rem}[dir=ltr] .md-source-file__fact .md-author{float:left}[dir=rtl] .md-source-file__fact .md-author{float:right}.md-source-file__fact .md-author{margin-right:.2rem}.md-source-file__fact svg{width:.9rem}:root{--md-status:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M11 9h2V7h-2m1 13c-4.41 0-8-3.59-8-8s3.59-8 8-8 8 3.59 8 8-3.59 8-8 8m0-18A10 10 0 0 0 2 12a10 10 0 0 0 10 10 10 10 0 0 0 10-10A10 10 0 0 0 12 2m-1 15h2v-6h-2v6Z"/></svg>');--md-status--new:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m23 12-2.44-2.78.34-3.68-3.61-.82-1.89-3.18L12 3 8.6 1.54 6.71 4.72l-3.61.81.34 3.68L1 12l2.44 2.78-.34 3.69 3.61.82 1.89 3.18L12 21l3.4 1.46 1.89-3.18 3.61-.82-.34-3.68L23 12m-10 5h-2v-2h2v2m0-4h-2V7h2v6Z"/></svg>');--md-status--deprecated:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9 3v1H4v2h1v13a2 2 0 0 0 2 2h10a2 2 0 0 0 2-2V6h1V4h-5V3H9m0 5h2v9H9V8m4 0h2v9h-2V8Z"/></svg>');--md-status--encrypted:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 1 3 5v6c0 5.55 3.84 10.74 9 12 5.16-1.26 9-6.45 9-12V5l-9-4m0 6c1.4 0 2.8 1.1 2.8 2.5V11c.6 0 1.2.6 1.2 1.3v3.5c0 .6-.6 1.2-1.3 1.2H9.2c-.6 0-1.2-.6-1.2-1.3v-3.5c0-.6.6-1.2 1.2-1.2V9.5C9.2 8.1 10.6 7 12 7m0 1.2c-.8 0-1.5.5-1.5 1.3V11h3V9.5c0-.8-.7-1.3-1.5-1.3Z"/></svg>')}.md-status:after{background-color:var(--md-default-fg-color--light);content:"";display:inline-block;height:1.125em;-webkit-mask-image:var(--md-status);mask-image:var(--md-status);-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;vertical-align:text-bottom;width:1.125em}.md-status:hover:after{background-color:currentcolor}.md-status--new:after{-webkit-mask-image:var(--md-status--new);mask-image:var(--md-status--new)}.md-status--deprecated:after{-webkit-mask-image:var(--md-status--deprecated);mask-image:var(--md-status--deprecated)}.md-status--encrypted:after{-webkit-mask-image:var(--md-status--encrypted);mask-image:var(--md-status--encrypted)}.md-tabs{background-color:var(--md-primary-fg-color);color:var(--md-primary-bg-color);display:block;line-height:1.3;overflow:auto;width:100%;z-index:3}@media print{.md-tabs{display:none}}@media screen and (max-width:76.234375em){.md-tabs{display:none}}.md-tabs[hidden]{pointer-events:none}[dir=ltr] .md-tabs__list{margin-left:.2rem}[dir=rtl] .md-tabs__list{margin-right:.2rem}.md-tabs__list{contain:content;display:flex;list-style:none;margin:0;overflow:auto;padding:0;scrollbar-width:none;white-space:nowrap}.md-tabs__list::-webkit-scrollbar{display:none}.md-tabs__item{height:2.4rem;padding-left:.6rem;padding-right:.6rem}.md-tabs__item--active .md-tabs__link{color:inherit;opacity:1}.md-tabs__link{-webkit-backface-visibility:hidden;backface-visibility:hidden;display:flex;font-size:.7rem;margin-top:.8rem;opacity:.7;outline-color:var(--md-accent-fg-color);outline-offset:.2rem;transition:transform .4s cubic-bezier(.1,.7,.1,1),opacity .25s}.md-tabs__link:focus,.md-tabs__link:hover{color:inherit;opacity:1}[dir=ltr] .md-tabs__link svg{margin-right:.4rem}[dir=rtl] .md-tabs__link svg{margin-left:.4rem}.md-tabs__link svg{fill:currentcolor;height:1.3em}.md-tabs__item:nth-child(2) .md-tabs__link{transition-delay:20ms}.md-tabs__item:nth-child(3) .md-tabs__link{transition-delay:40ms}.md-tabs__item:nth-child(4) .md-tabs__link{transition-delay:60ms}.md-tabs__item:nth-child(5) .md-tabs__link{transition-delay:80ms}.md-tabs__item:nth-child(6) .md-tabs__link{transition-delay:.1s}.md-tabs__item:nth-child(7) .md-tabs__link{transition-delay:.12s}.md-tabs__item:nth-child(8) .md-tabs__link{transition-delay:.14s}.md-tabs__item:nth-child(9) .md-tabs__link{transition-delay:.16s}.md-tabs__item:nth-child(10) .md-tabs__link{transition-delay:.18s}.md-tabs__item:nth-child(11) .md-tabs__link{transition-delay:.2s}.md-tabs__item:nth-child(12) .md-tabs__link{transition-delay:.22s}.md-tabs__item:nth-child(13) .md-tabs__link{transition-delay:.24s}.md-tabs__item:nth-child(14) .md-tabs__link{transition-delay:.26s}.md-tabs__item:nth-child(15) .md-tabs__link{transition-delay:.28s}.md-tabs__item:nth-child(16) .md-tabs__link{transition-delay:.3s}.md-tabs[hidden] .md-tabs__link{opacity:0;transform:translateY(50%);transition:transform 0ms .1s,opacity .1s}:root{--md-tag-icon:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m5.41 21 .71-4h-4l.35-2h4l1.06-6h-4l.35-2h4l.71-4h2l-.71 4h6l.71-4h2l-.71 4h4l-.35 2h-4l-1.06 6h4l-.35 2h-4l-.71 4h-2l.71-4h-6l-.71 4h-2M9.53 9l-1.06 6h6l1.06-6h-6Z"/></svg>')}.md-typeset .md-tags:not([hidden]){display:inline-flex;flex-wrap:wrap;gap:.5em;margin-bottom:.75em;margin-top:-.125em}.md-typeset .md-tag{align-items:center;background:var(--md-default-fg-color--lightest);border-radius:2.4rem;display:inline-flex;font-size:.64rem;font-size:min(.8em,.64rem);font-weight:700;gap:.5em;letter-spacing:normal;line-height:1.6;padding:.3125em .78125em}.md-typeset .md-tag[href]{-webkit-tap-highlight-color:transparent;color:inherit;outline:none;transition:color 125ms,background-color 125ms}.md-typeset .md-tag[href]:focus,.md-typeset .md-tag[href]:hover{background-color:var(--md-accent-fg-color);color:var(--md-accent-bg-color)}[id]>.md-typeset .md-tag{vertical-align:text-top}.md-typeset .md-tag-icon:before{background-color:var(--md-default-fg-color--lighter);content:"";display:inline-block;height:1.2em;-webkit-mask-image:var(--md-tag-icon);mask-image:var(--md-tag-icon);-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;transition:background-color 125ms;vertical-align:text-bottom;width:1.2em}.md-typeset .md-tag-icon[href]:focus:before,.md-typeset .md-tag-icon[href]:hover:before{background-color:var(--md-accent-bg-color)}@keyframes pulse{0%{transform:scale(.95)}75%{transform:scale(1)}to{transform:scale(.95)}}:root{--md-annotation-bg-icon:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 2A10 10 0 0 0 2 12a10 10 0 0 0 10 10 10 10 0 0 0 10-10A10 10 0 0 0 12 2Z"/></svg>');--md-annotation-icon:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 13h-4v4h-2v-4H7v-2h4V7h2v4h4m-5-9A10 10 0 0 0 2 12a10 10 0 0 0 10 10 10 10 0 0 0 10-10A10 10 0 0 0 12 2Z"/></svg>');--md-tooltip-width:20rem}.md-tooltip{-webkit-backface-visibility:hidden;backface-visibility:hidden;background-color:var(--md-default-bg-color);border-radius:.1rem;box-shadow:var(--md-shadow-z2);color:var(--md-default-fg-color);font-family:var(--md-text-font-family);left:clamp(var(--md-tooltip-0,0rem) + .8rem,var(--md-tooltip-x),100vw + var(--md-tooltip-0,0rem) + .8rem - var(--md-tooltip-width) - 2 * .8rem);max-width:calc(100vw - 1.6rem);opacity:0;position:absolute;top:var(--md-tooltip-y);transform:translateY(-.4rem);transition:transform 0ms .25s,opacity .25s,z-index .25s;width:var(--md-tooltip-width);z-index:0}.md-tooltip--active{opacity:1;transform:translateY(0);transition:transform .25s cubic-bezier(.1,.7,.1,1),opacity .25s,z-index 0ms;z-index:2}.md-tooltip--inline{font-weight:700;-webkit-user-select:none;user-select:none;width:auto}.md-tooltip--inline:not(.md-tooltip--active){transform:translateY(.2rem) scale(.9)}.md-tooltip--inline .md-tooltip__inner{font-size:.5rem;padding:.2rem .4rem}[hidden]+.md-tooltip--inline{display:none}.focus-visible>.md-tooltip,.md-tooltip:target{outline:var(--md-accent-fg-color) auto}.md-tooltip__inner{font-size:.64rem;padding:.8rem}.md-tooltip__inner.md-typeset>:first-child{margin-top:0}.md-tooltip__inner.md-typeset>:last-child{margin-bottom:0}.md-annotation{font-weight:400;outline:none;vertical-align:text-bottom;white-space:normal}[dir=rtl] .md-annotation{direction:rtl}code .md-annotation{font-family:var(--md-code-font-family);font-size:inherit}.md-annotation:not([hidden]){display:inline-block;line-height:1.25}.md-annotation__index{border-radius:.01px;cursor:pointer;display:inline-block;margin-left:.4ch;margin-right:.4ch;outline:none;overflow:hidden;position:relative;-webkit-user-select:none;user-select:none;vertical-align:text-top;z-index:0}.md-annotation .md-annotation__index{transition:z-index .25s}@media screen{.md-annotation__index{width:2.2ch}[data-md-visible]>.md-annotation__index{animation:pulse 2s infinite}.md-annotation__index:before{background:var(--md-default-bg-color);-webkit-mask-image:var(--md-annotation-bg-icon);mask-image:var(--md-annotation-bg-icon)}.md-annotation__index:after,.md-annotation__index:before{content:"";height:2.2ch;-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;position:absolute;top:-.1ch;width:2.2ch;z-index:-1}.md-annotation__index:after{background-color:var(--md-default-fg-color--lighter);-webkit-mask-image:var(--md-annotation-icon);mask-image:var(--md-annotation-icon);transform:scale(1.0001);transition:background-color .25s,transform .25s}.md-tooltip--active+.md-annotation__index:after{transform:rotate(45deg)}.md-tooltip--active+.md-annotation__index:after,:hover>.md-annotation__index:after{background-color:var(--md-accent-fg-color)}}.md-tooltip--active+.md-annotation__index{animation-play-state:paused;transition-duration:0ms;z-index:2}.md-annotation__index [data-md-annotation-id]{display:inline-block}@media print{.md-annotation__index [data-md-annotation-id]{background:var(--md-default-fg-color--lighter);border-radius:2ch;color:var(--md-default-bg-color);font-weight:700;padding:0 .6ch;white-space:nowrap}.md-annotation__index [data-md-annotation-id]:after{content:attr(data-md-annotation-id)}}.md-typeset .md-annotation-list{counter-reset:xxx;list-style:none}.md-typeset .md-annotation-list li{position:relative}[dir=ltr] .md-typeset .md-annotation-list li:before{left:-2.125em}[dir=rtl] .md-typeset .md-annotation-list li:before{right:-2.125em}.md-typeset .md-annotation-list li:before{background:var(--md-default-fg-color--lighter);border-radius:2ch;color:var(--md-default-bg-color);content:counter(xxx);counter-increment:xxx;font-size:.8875em;font-weight:700;height:2ch;line-height:1.25;min-width:2ch;padding:0 .6ch;position:absolute;text-align:center;top:.25em}[dir=ltr] .md-top{margin-left:50%}[dir=rtl] .md-top{margin-right:50%}.md-top{background-color:var(--md-default-bg-color);border-radius:1.6rem;box-shadow:var(--md-shadow-z2);color:var(--md-default-fg-color--light);cursor:pointer;display:block;font-size:.7rem;outline:none;padding:.4rem .8rem;position:fixed;top:3.2rem;transform:translate(-50%);transition:color 125ms,background-color 125ms,transform 125ms cubic-bezier(.4,0,.2,1),opacity 125ms;z-index:2}@media print{.md-top{display:none}}[dir=rtl] .md-top{transform:translate(50%)}.md-top[hidden]{opacity:0;pointer-events:none;transform:translate(-50%,.2rem);transition-duration:0ms}[dir=rtl] .md-top[hidden]{transform:translate(50%,.2rem)}.md-top:focus,.md-top:hover{background-color:var(--md-accent-fg-color);color:var(--md-accent-bg-color)}.md-top svg{display:inline-block;vertical-align:-.5em}@keyframes hoverfix{0%{pointer-events:none}}:root{--md-version-icon:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 320 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M137.4 374.6c12.5 12.5 32.8 12.5 45.3 0l128-128c9.2-9.2 11.9-22.9 6.9-34.9S301 191.9 288 191.9L32 192c-12.9 0-24.6 7.8-29.6 19.8s-2.2 25.7 6.9 34.9l128 128z"/></svg>')}.md-version{flex-shrink:0;font-size:.8rem;height:2.4rem}[dir=ltr] .md-version__current{margin-left:1.4rem;margin-right:.4rem}[dir=rtl] .md-version__current{margin-left:.4rem;margin-right:1.4rem}.md-version__current{color:inherit;cursor:pointer;outline:none;position:relative;top:.05rem}[dir=ltr] .md-version__current:after{margin-left:.4rem}[dir=rtl] .md-version__current:after{margin-right:.4rem}.md-version__current:after{background-color:currentcolor;content:"";display:inline-block;height:.6rem;-webkit-mask-image:var(--md-version-icon);mask-image:var(--md-version-icon);-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;width:.4rem}.md-version__list{background-color:var(--md-default-bg-color);border-radius:.1rem;box-shadow:var(--md-shadow-z2);color:var(--md-default-fg-color);list-style-type:none;margin:.2rem .8rem;max-height:0;opacity:0;overflow:auto;padding:0;position:absolute;scroll-snap-type:y mandatory;top:.15rem;transition:max-height 0ms .5s,opacity .25s .25s;z-index:3}.md-version:focus-within .md-version__list,.md-version:hover .md-version__list{max-height:10rem;opacity:1;transition:max-height 0ms,opacity .25s}@media (hover:none),(pointer:coarse){.md-version:hover .md-version__list{animation:hoverfix .25s forwards}.md-version:focus-within .md-version__list{animation:none}}.md-version__item{line-height:1.8rem}[dir=ltr] .md-version__link{padding-left:.6rem;padding-right:1.2rem}[dir=rtl] .md-version__link{padding-left:1.2rem;padding-right:.6rem}.md-version__link{cursor:pointer;display:block;outline:none;scroll-snap-align:start;transition:color .25s,background-color .25s;white-space:nowrap;width:100%}.md-version__link:focus,.md-version__link:hover{color:var(--md-accent-fg-color)}.md-version__link:focus{background-color:var(--md-default-fg-color--lightest)}:root{--md-admonition-icon--note:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 2C6.47 2 2 6.47 2 12s4.47 10 10 10 10-4.47 10-10S17.53 2 12 2m3.1 5.07c.14 0 .28.05.4.16l1.27 1.27c.23.22.23.57 0 .78l-1 1-2.05-2.05 1-1c.1-.11.24-.16.38-.16m-1.97 1.74 2.06 2.06-6.06 6.06H7.07v-2.06l6.06-6.06Z"/></svg>');--md-admonition-icon--abstract:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 9H7V7h10m0 6H7v-2h10m-3 6H7v-2h7M12 3a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m7 0h-4.18C14.4 1.84 13.3 1 12 1c-1.3 0-2.4.84-2.82 2H5a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2V5a2 2 0 0 0-2-2Z"/></svg>');--md-admonition-icon--info:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 9h-2V7h2m0 10h-2v-6h2m-1-9A10 10 0 0 0 2 12a10 10 0 0 0 10 10 10 10 0 0 0 10-10A10 10 0 0 0 12 2Z"/></svg>');--md-admonition-icon--tip:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17.66 11.2c-.23-.3-.51-.56-.77-.82-.67-.6-1.43-1.03-2.07-1.66C13.33 7.26 13 4.85 13.95 3c-.95.23-1.78.75-2.49 1.32-2.59 2.08-3.61 5.75-2.39 8.9.04.1.08.2.08.33 0 .22-.15.42-.35.5-.23.1-.47.04-.66-.12a.58.58 0 0 1-.14-.17c-1.13-1.43-1.31-3.48-.55-5.12C5.78 10 4.87 12.3 5 14.47c.06.5.12 1 .29 1.5.14.6.41 1.2.71 1.73 1.08 1.73 2.95 2.97 4.96 3.22 2.14.27 4.43-.12 6.07-1.6 1.83-1.66 2.47-4.32 1.53-6.6l-.13-.26c-.21-.46-.77-1.26-.77-1.26m-3.16 6.3c-.28.24-.74.5-1.1.6-1.12.4-2.24-.16-2.9-.82 1.19-.28 1.9-1.16 2.11-2.05.17-.8-.15-1.46-.28-2.23-.12-.74-.1-1.37.17-2.06.19.38.39.76.63 1.06.77 1 1.98 1.44 2.24 2.8.04.14.06.28.06.43.03.82-.33 1.72-.93 2.27Z"/></svg>');--md-admonition-icon--success:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 7 9 19l-5.5-5.5 1.41-1.41L9 16.17 19.59 5.59 21 7Z"/></svg>');--md-admonition-icon--question:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m15.07 11.25-.9.92C13.45 12.89 13 13.5 13 15h-2v-.5c0-1.11.45-2.11 1.17-2.83l1.24-1.26c.37-.36.59-.86.59-1.41a2 2 0 0 0-2-2 2 2 0 0 0-2 2H8a4 4 0 0 1 4-4 4 4 0 0 1 4 4 3.2 3.2 0 0 1-.93 2.25M13 19h-2v-2h2M12 2A10 10 0 0 0 2 12a10 10 0 0 0 10 10 10 10 0 0 0 10-10c0-5.53-4.5-10-10-10Z"/></svg>');--md-admonition-icon--warning:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 14h-2V9h2m0 9h-2v-2h2M1 21h22L12 2 1 21Z"/></svg>');--md-admonition-icon--failure:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>');--md-admonition-icon--danger:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m11.5 20 4.86-9.73H13V4l-5 9.73h3.5V20M12 2c2.75 0 5.1 1 7.05 2.95C21 6.9 22 9.25 22 12s-1 5.1-2.95 7.05C17.1 21 14.75 22 12 22s-5.1-1-7.05-2.95C3 17.1 2 14.75 2 12s1-5.1 2.95-7.05C6.9 3 9.25 2 12 2Z"/></svg>');--md-admonition-icon--bug:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M11 13h2v1h-2v-1m10-8v6c0 5.5-3.8 10.7-9 12-5.2-1.3-9-6.5-9-12V5l9-4 9 4m-4 5h-2.2c-.2-.6-.6-1.1-1.1-1.5l1.2-1.2-.7-.7L12.8 8H12c-.2 0-.5 0-.7.1L9.9 6.6l-.8.8 1.2 1.2c-.5.3-.9.8-1.1 1.4H7v1h2v1H7v1h2v1H7v1h2.2c.4 1.2 1.5 2 2.8 2s2.4-.8 2.8-2H17v-1h-2v-1h2v-1h-2v-1h2v-1m-6 2h2v-1h-2v1Z"/></svg>');--md-admonition-icon--example:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M7 2v2h1v14a4 4 0 0 0 4 4 4 4 0 0 0 4-4V4h1V2H7m4 14c-.6 0-1-.4-1-1s.4-1 1-1 1 .4 1 1-.4 1-1 1m2-4c-.6 0-1-.4-1-1s.4-1 1-1 1 .4 1 1-.4 1-1 1m1-5h-4V4h4v3Z"/></svg>');--md-admonition-icon--quote:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14 17h3l2-4V7h-6v6h3M6 17h3l2-4V7H5v6h3l-2 4Z"/></svg>')}.md-typeset .admonition,.md-typeset details{background-color:var(--md-admonition-bg-color);border:.075rem solid #448aff;border-radius:.2rem;box-shadow:var(--md-shadow-z1);color:var(--md-admonition-fg-color);display:flow-root;font-size:.64rem;margin:1.5625em 0;padding:0 .6rem;page-break-inside:avoid;transition:box-shadow 125ms}@media print{.md-typeset .admonition,.md-typeset details{box-shadow:none}}.md-typeset .admonition:focus-within,.md-typeset details:focus-within{box-shadow:0 0 0 .2rem #448aff1a}.md-typeset .admonition>*,.md-typeset details>*{box-sizing:border-box}.md-typeset .admonition .admonition,.md-typeset .admonition details,.md-typeset details .admonition,.md-typeset details details{margin-bottom:1em;margin-top:1em}.md-typeset .admonition .md-typeset__scrollwrap,.md-typeset details .md-typeset__scrollwrap{margin:1em -.6rem}.md-typeset .admonition .md-typeset__table,.md-typeset details .md-typeset__table{padding:0 .6rem}.md-typeset .admonition>.tabbed-set:only-child,.md-typeset details>.tabbed-set:only-child{margin-top:0}html .md-typeset .admonition>:last-child,html .md-typeset details>:last-child{margin-bottom:.6rem}[dir=ltr] .md-typeset .admonition-title,[dir=ltr] .md-typeset summary{padding-left:2rem;padding-right:.6rem}[dir=rtl] .md-typeset .admonition-title,[dir=rtl] .md-typeset summary{padding-left:.6rem;padding-right:2rem}[dir=ltr] .md-typeset .admonition-title,[dir=ltr] .md-typeset summary{border-left-width:.2rem}[dir=rtl] .md-typeset .admonition-title,[dir=rtl] .md-typeset summary{border-right-width:.2rem}[dir=ltr] .md-typeset .admonition-title,[dir=ltr] .md-typeset summary{border-top-left-radius:.1rem}[dir=ltr] .md-typeset .admonition-title,[dir=ltr] .md-typeset summary,[dir=rtl] .md-typeset .admonition-title,[dir=rtl] .md-typeset summary{border-top-right-radius:.1rem}[dir=rtl] .md-typeset .admonition-title,[dir=rtl] .md-typeset summary{border-top-left-radius:.1rem}.md-typeset .admonition-title,.md-typeset summary{background-color:#448aff1a;border:none;font-weight:700;margin:0 -.6rem;padding-bottom:.4rem;padding-top:.4rem;position:relative}html .md-typeset .admonition-title:last-child,html .md-typeset summary:last-child{margin-bottom:0}[dir=ltr] .md-typeset .admonition-title:before,[dir=ltr] .md-typeset summary:before{left:.6rem}[dir=rtl] .md-typeset .admonition-title:before,[dir=rtl] .md-typeset summary:before{right:.6rem}.md-typeset .admonition-title:before,.md-typeset summary:before{background-color:#448aff;content:"";height:1rem;-webkit-mask-image:var(--md-admonition-icon--note);mask-image:var(--md-admonition-icon--note);-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;position:absolute;top:.625em;width:1rem}.md-typeset .admonition-title code,.md-typeset summary code{box-shadow:0 0 0 .05rem var(--md-default-fg-color--lightest)}.md-typeset .admonition.note,.md-typeset details.note{border-color:#448aff}.md-typeset .admonition.note:focus-within,.md-typeset details.note:focus-within{box-shadow:0 0 0 .2rem #448aff1a}.md-typeset .note>.admonition-title,.md-typeset .note>summary{background-color:#448aff1a}.md-typeset .note>.admonition-title:before,.md-typeset .note>summary:before{background-color:#448aff;-webkit-mask-image:var(--md-admonition-icon--note);mask-image:var(--md-admonition-icon--note)}.md-typeset .note>.admonition-title:after,.md-typeset .note>summary:after{color:#448aff}.md-typeset .admonition.abstract,.md-typeset details.abstract{border-color:#00b0ff}.md-typeset .admonition.abstract:focus-within,.md-typeset details.abstract:focus-within{box-shadow:0 0 0 .2rem #00b0ff1a}.md-typeset .abstract>.admonition-title,.md-typeset .abstract>summary{background-color:#00b0ff1a}.md-typeset .abstract>.admonition-title:before,.md-typeset .abstract>summary:before{background-color:#00b0ff;-webkit-mask-image:var(--md-admonition-icon--abstract);mask-image:var(--md-admonition-icon--abstract)}.md-typeset .abstract>.admonition-title:after,.md-typeset .abstract>summary:after{color:#00b0ff}.md-typeset .admonition.info,.md-typeset details.info{border-color:#00b8d4}.md-typeset .admonition.info:focus-within,.md-typeset details.info:focus-within{box-shadow:0 0 0 .2rem #00b8d41a}.md-typeset .info>.admonition-title,.md-typeset .info>summary{background-color:#00b8d41a}.md-typeset .info>.admonition-title:before,.md-typeset .info>summary:before{background-color:#00b8d4;-webkit-mask-image:var(--md-admonition-icon--info);mask-image:var(--md-admonition-icon--info)}.md-typeset .info>.admonition-title:after,.md-typeset .info>summary:after{color:#00b8d4}.md-typeset .admonition.tip,.md-typeset details.tip{border-color:#00bfa5}.md-typeset .admonition.tip:focus-within,.md-typeset details.tip:focus-within{box-shadow:0 0 0 .2rem #00bfa51a}.md-typeset .tip>.admonition-title,.md-typeset .tip>summary{background-color:#00bfa51a}.md-typeset .tip>.admonition-title:before,.md-typeset .tip>summary:before{background-color:#00bfa5;-webkit-mask-image:var(--md-admonition-icon--tip);mask-image:var(--md-admonition-icon--tip)}.md-typeset .tip>.admonition-title:after,.md-typeset .tip>summary:after{color:#00bfa5}.md-typeset .admonition.success,.md-typeset details.success{border-color:#00c853}.md-typeset .admonition.success:focus-within,.md-typeset details.success:focus-within{box-shadow:0 0 0 .2rem #00c8531a}.md-typeset .success>.admonition-title,.md-typeset .success>summary{background-color:#00c8531a}.md-typeset .success>.admonition-title:before,.md-typeset .success>summary:before{background-color:#00c853;-webkit-mask-image:var(--md-admonition-icon--success);mask-image:var(--md-admonition-icon--success)}.md-typeset .success>.admonition-title:after,.md-typeset .success>summary:after{color:#00c853}.md-typeset .admonition.question,.md-typeset details.question{border-color:#64dd17}.md-typeset .admonition.question:focus-within,.md-typeset details.question:focus-within{box-shadow:0 0 0 .2rem #64dd171a}.md-typeset .question>.admonition-title,.md-typeset .question>summary{background-color:#64dd171a}.md-typeset .question>.admonition-title:before,.md-typeset .question>summary:before{background-color:#64dd17;-webkit-mask-image:var(--md-admonition-icon--question);mask-image:var(--md-admonition-icon--question)}.md-typeset .question>.admonition-title:after,.md-typeset .question>summary:after{color:#64dd17}.md-typeset .admonition.warning,.md-typeset details.warning{border-color:#ff9100}.md-typeset .admonition.warning:focus-within,.md-typeset details.warning:focus-within{box-shadow:0 0 0 .2rem #ff91001a}.md-typeset .warning>.admonition-title,.md-typeset .warning>summary{background-color:#ff91001a}.md-typeset .warning>.admonition-title:before,.md-typeset .warning>summary:before{background-color:#ff9100;-webkit-mask-image:var(--md-admonition-icon--warning);mask-image:var(--md-admonition-icon--warning)}.md-typeset .warning>.admonition-title:after,.md-typeset .warning>summary:after{color:#ff9100}.md-typeset .admonition.failure,.md-typeset details.failure{border-color:#ff5252}.md-typeset .admonition.failure:focus-within,.md-typeset details.failure:focus-within{box-shadow:0 0 0 .2rem #ff52521a}.md-typeset .failure>.admonition-title,.md-typeset .failure>summary{background-color:#ff52521a}.md-typeset .failure>.admonition-title:before,.md-typeset .failure>summary:before{background-color:#ff5252;-webkit-mask-image:var(--md-admonition-icon--failure);mask-image:var(--md-admonition-icon--failure)}.md-typeset .failure>.admonition-title:after,.md-typeset .failure>summary:after{color:#ff5252}.md-typeset .admonition.danger,.md-typeset details.danger{border-color:#ff1744}.md-typeset .admonition.danger:focus-within,.md-typeset details.danger:focus-within{box-shadow:0 0 0 .2rem #ff17441a}.md-typeset .danger>.admonition-title,.md-typeset .danger>summary{background-color:#ff17441a}.md-typeset .danger>.admonition-title:before,.md-typeset .danger>summary:before{background-color:#ff1744;-webkit-mask-image:var(--md-admonition-icon--danger);mask-image:var(--md-admonition-icon--danger)}.md-typeset .danger>.admonition-title:after,.md-typeset .danger>summary:after{color:#ff1744}.md-typeset .admonition.bug,.md-typeset details.bug{border-color:#f50057}.md-typeset .admonition.bug:focus-within,.md-typeset details.bug:focus-within{box-shadow:0 0 0 .2rem #f500571a}.md-typeset .bug>.admonition-title,.md-typeset .bug>summary{background-color:#f500571a}.md-typeset .bug>.admonition-title:before,.md-typeset .bug>summary:before{background-color:#f50057;-webkit-mask-image:var(--md-admonition-icon--bug);mask-image:var(--md-admonition-icon--bug)}.md-typeset .bug>.admonition-title:after,.md-typeset .bug>summary:after{color:#f50057}.md-typeset .admonition.example,.md-typeset details.example{border-color:#7c4dff}.md-typeset .admonition.example:focus-within,.md-typeset details.example:focus-within{box-shadow:0 0 0 .2rem #7c4dff1a}.md-typeset .example>.admonition-title,.md-typeset .example>summary{background-color:#7c4dff1a}.md-typeset .example>.admonition-title:before,.md-typeset .example>summary:before{background-color:#7c4dff;-webkit-mask-image:var(--md-admonition-icon--example);mask-image:var(--md-admonition-icon--example)}.md-typeset .example>.admonition-title:after,.md-typeset .example>summary:after{color:#7c4dff}.md-typeset .admonition.quote,.md-typeset details.quote{border-color:#9e9e9e}.md-typeset .admonition.quote:focus-within,.md-typeset details.quote:focus-within{box-shadow:0 0 0 .2rem #9e9e9e1a}.md-typeset .quote>.admonition-title,.md-typeset .quote>summary{background-color:#9e9e9e1a}.md-typeset .quote>.admonition-title:before,.md-typeset .quote>summary:before{background-color:#9e9e9e;-webkit-mask-image:var(--md-admonition-icon--quote);mask-image:var(--md-admonition-icon--quote)}.md-typeset .quote>.admonition-title:after,.md-typeset .quote>summary:after{color:#9e9e9e}:root{--md-footnotes-icon:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 7v4H5.83l3.58-3.59L8 6l-6 6 6 6 1.41-1.42L5.83 13H21V7h-2Z"/></svg>')}.md-typeset .footnote{color:var(--md-default-fg-color--light);font-size:.64rem}[dir=ltr] .md-typeset .footnote>ol{margin-left:0}[dir=rtl] .md-typeset .footnote>ol{margin-right:0}.md-typeset .footnote>ol>li{transition:color 125ms}.md-typeset .footnote>ol>li:target{color:var(--md-default-fg-color)}.md-typeset .footnote>ol>li:focus-within .footnote-backref{opacity:1;transform:translateX(0);transition:none}.md-typeset .footnote>ol>li:hover .footnote-backref,.md-typeset .footnote>ol>li:target .footnote-backref{opacity:1;transform:translateX(0)}.md-typeset .footnote>ol>li>:first-child{margin-top:0}.md-typeset .footnote-ref{font-size:.75em;font-weight:700}html .md-typeset .footnote-ref{outline-offset:.1rem}.md-typeset [id^="fnref:"]:target>.footnote-ref{outline:auto}.md-typeset .footnote-backref{color:var(--md-typeset-a-color);display:inline-block;font-size:0;opacity:0;transform:translateX(.25rem);transition:color .25s,transform .25s .25s,opacity 125ms .25s;vertical-align:text-bottom}@media print{.md-typeset .footnote-backref{color:var(--md-typeset-a-color);opacity:1;transform:translateX(0)}}[dir=rtl] .md-typeset .footnote-backref{transform:translateX(-.25rem)}.md-typeset .footnote-backref:hover{color:var(--md-accent-fg-color)}.md-typeset .footnote-backref:before{background-color:currentcolor;content:"";display:inline-block;height:.8rem;-webkit-mask-image:var(--md-footnotes-icon);mask-image:var(--md-footnotes-icon);-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;width:.8rem}[dir=rtl] .md-typeset .footnote-backref:before svg{transform:scaleX(-1)}[dir=ltr] .md-typeset .headerlink{margin-left:.5rem}[dir=rtl] .md-typeset .headerlink{margin-right:.5rem}.md-typeset .headerlink{color:var(--md-default-fg-color--lighter);display:inline-block;opacity:0;transition:color .25s,opacity 125ms}@media print{.md-typeset .headerlink{display:none}}.md-typeset .headerlink:focus,.md-typeset :hover>.headerlink,.md-typeset :target>.headerlink{opacity:1;transition:color .25s,opacity 125ms}.md-typeset .headerlink:focus,.md-typeset .headerlink:hover,.md-typeset :target>.headerlink{color:var(--md-accent-fg-color)}.md-typeset :target{--md-scroll-margin:3.6rem;--md-scroll-offset:0rem;scroll-margin-top:calc(var(--md-scroll-margin) - var(--md-scroll-offset))}@media screen and (min-width:76.25em){.md-header--lifted~.md-container .md-typeset :target{--md-scroll-margin:6rem}}.md-typeset h1:target,.md-typeset h2:target,.md-typeset h3:target{--md-scroll-offset:0.2rem}.md-typeset h4:target{--md-scroll-offset:0.15rem}.md-typeset div.arithmatex{overflow:auto}@media screen and (max-width:44.984375em){.md-typeset div.arithmatex{margin:0 -.8rem}.md-typeset div.arithmatex>*{width:-webkit-min-content;width:min-content}}.md-typeset div.arithmatex>*{margin-left:auto!important;margin-right:auto!important;padding:0 .8rem;touch-action:auto}.md-typeset div.arithmatex>* mjx-container{margin:0!important}.md-typeset div.arithmatex mjx-assistive-mml{height:0}.md-typeset del.critic{background-color:var(--md-typeset-del-color)}.md-typeset del.critic,.md-typeset ins.critic{-webkit-box-decoration-break:clone;box-decoration-break:clone}.md-typeset ins.critic{background-color:var(--md-typeset-ins-color)}.md-typeset .critic.comment{-webkit-box-decoration-break:clone;box-decoration-break:clone;color:var(--md-code-hl-comment-color)}.md-typeset .critic.comment:before{content:"/* "}.md-typeset .critic.comment:after{content:" */"}.md-typeset .critic.block{box-shadow:none;display:block;margin:1em 0;overflow:auto;padding-left:.8rem;padding-right:.8rem}.md-typeset .critic.block>:first-child{margin-top:.5em}.md-typeset .critic.block>:last-child{margin-bottom:.5em}:root{--md-details-icon:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M8.59 16.58 13.17 12 8.59 7.41 10 6l6 6-6 6-1.41-1.42Z"/></svg>')}.md-typeset details{display:flow-root;overflow:visible;padding-top:0}.md-typeset details[open]>summary:after{transform:rotate(90deg)}.md-typeset details:not([open]){box-shadow:none;padding-bottom:0}.md-typeset details:not([open])>summary{border-radius:.1rem}[dir=ltr] .md-typeset summary{padding-right:1.8rem}[dir=rtl] .md-typeset summary{padding-left:1.8rem}[dir=ltr] .md-typeset summary{border-top-left-radius:.1rem}[dir=ltr] .md-typeset summary,[dir=rtl] .md-typeset summary{border-top-right-radius:.1rem}[dir=rtl] .md-typeset summary{border-top-left-radius:.1rem}.md-typeset summary{cursor:pointer;display:block;min-height:1rem;overflow:hidden}.md-typeset summary.focus-visible{outline-color:var(--md-accent-fg-color);outline-offset:.2rem}.md-typeset summary:not(.focus-visible){-webkit-tap-highlight-color:transparent;outline:none}[dir=ltr] .md-typeset summary:after{right:.4rem}[dir=rtl] .md-typeset summary:after{left:.4rem}.md-typeset summary:after{background-color:currentcolor;content:"";height:1rem;-webkit-mask-image:var(--md-details-icon);mask-image:var(--md-details-icon);-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;position:absolute;top:.625em;transform:rotate(0deg);transition:transform .25s;width:1rem}[dir=rtl] .md-typeset summary:after{transform:rotate(180deg)}.md-typeset summary::marker{display:none}.md-typeset summary::-webkit-details-marker{display:none}.md-typeset .emojione,.md-typeset .gemoji,.md-typeset .twemoji{--md-icon-size:1.125em;display:inline-flex;height:var(--md-icon-size);vertical-align:text-top}.md-typeset .emojione svg,.md-typeset .gemoji svg,.md-typeset .twemoji svg{fill:currentcolor;max-height:100%;width:var(--md-icon-size)}.md-typeset .lg,.md-typeset .xl,.md-typeset .xxl,.md-typeset .xxxl{vertical-align:text-bottom}.md-typeset .middle{vertical-align:middle}.md-typeset .lg{--md-icon-size:1.5em}.md-typeset .xl{--md-icon-size:2.25em}.md-typeset .xxl{--md-icon-size:3em}.md-typeset .xxxl{--md-icon-size:4em}.highlight .o,.highlight .ow{color:var(--md-code-hl-operator-color)}.highlight .p{color:var(--md-code-hl-punctuation-color)}.highlight .cpf,.highlight .l,.highlight .s,.highlight .s1,.highlight .s2,.highlight .sb,.highlight .sc,.highlight .si,.highlight .ss{color:var(--md-code-hl-string-color)}.highlight .cp,.highlight .se,.highlight .sh,.highlight .sr,.highlight .sx{color:var(--md-code-hl-special-color)}.highlight .il,.highlight .m,.highlight .mb,.highlight .mf,.highlight .mh,.highlight .mi,.highlight .mo{color:var(--md-code-hl-number-color)}.highlight .k,.highlight .kd,.highlight .kn,.highlight .kp,.highlight .kr,.highlight .kt{color:var(--md-code-hl-keyword-color)}.highlight .kc,.highlight .n{color:var(--md-code-hl-name-color)}.highlight .bp,.highlight .nb,.highlight .no{color:var(--md-code-hl-constant-color)}.highlight .nc,.highlight .ne,.highlight .nf,.highlight .nn{color:var(--md-code-hl-function-color)}.highlight .nd,.highlight .ni,.highlight .nl,.highlight .nt{color:var(--md-code-hl-keyword-color)}.highlight .c,.highlight .c1,.highlight .ch,.highlight .cm,.highlight .cs,.highlight .sd{color:var(--md-code-hl-comment-color)}.highlight .na,.highlight .nv,.highlight .vc,.highlight .vg,.highlight .vi{color:var(--md-code-hl-variable-color)}.highlight .ge,.highlight .gh,.highlight .go,.highlight .gp,.highlight .gr,.highlight .gs,.highlight .gt,.highlight .gu{color:var(--md-code-hl-generic-color)}.highlight .gd,.highlight .gi{border-radius:.1rem;margin:0 -.125em;padding:0 .125em}.highlight .gd{background-color:var(--md-typeset-del-color)}.highlight .gi{background-color:var(--md-typeset-ins-color)}.highlight .hll{background-color:var(--md-code-hl-color--light);box-shadow:2px 0 0 0 var(--md-code-hl-color) inset;display:block;margin:0 -1.1764705882em;padding:0 1.1764705882em}.highlight span.filename{background-color:var(--md-code-bg-color);border-bottom:.05rem solid var(--md-default-fg-color--lightest);border-top-left-radius:.1rem;border-top-right-radius:.1rem;display:flow-root;font-size:.85em;font-weight:700;margin-top:1em;padding:.6617647059em 1.1764705882em;position:relative}.highlight span.filename+pre{margin-top:0}.highlight span.filename+pre>code{border-top-left-radius:0;border-top-right-radius:0}.highlight [data-linenos]:before{background-color:var(--md-code-bg-color);box-shadow:-.05rem 0 var(--md-default-fg-color--lightest) inset;color:var(--md-default-fg-color--light);content:attr(data-linenos);float:left;left:-1.1764705882em;margin-left:-1.1764705882em;margin-right:1.1764705882em;padding-left:1.1764705882em;position:sticky;-webkit-user-select:none;user-select:none;z-index:3}.highlight code a[id]{position:absolute;visibility:hidden}.highlight code[data-md-copying] .hll{display:contents}.highlight code[data-md-copying] .md-annotation{display:none}.highlighttable{display:flow-root}.highlighttable tbody,.highlighttable td{display:block;padding:0}.highlighttable tr{display:flex}.highlighttable pre{margin:0}.highlighttable th.filename{flex-grow:1;padding:0;text-align:left}.highlighttable th.filename span.filename{margin-top:0}.highlighttable .linenos{background-color:var(--md-code-bg-color);border-bottom-left-radius:.1rem;border-top-left-radius:.1rem;font-size:.85em;padding:.7720588235em 0 .7720588235em 1.1764705882em;-webkit-user-select:none;user-select:none}.highlighttable .linenodiv{box-shadow:-.05rem 0 var(--md-default-fg-color--lightest) inset;padding-right:.5882352941em}.highlighttable .linenodiv pre{color:var(--md-default-fg-color--light);text-align:right}.highlighttable .code{flex:1;min-width:0}.linenodiv a{color:inherit}.md-typeset .highlighttable{direction:ltr;margin:1em 0}.md-typeset .highlighttable>tbody>tr>.code>div>pre>code{border-bottom-left-radius:0;border-top-left-radius:0}.md-typeset .highlight+.result{border:.05rem solid var(--md-code-bg-color);border-bottom-left-radius:.1rem;border-bottom-right-radius:.1rem;border-top-width:.1rem;margin-top:-1.125em;overflow:visible;padding:0 1em}.md-typeset .highlight+.result:after{clear:both;content:"";display:block}@media screen and (max-width:44.984375em){.md-content__inner>.highlight{margin:1em -.8rem}.md-content__inner>.highlight>.filename,.md-content__inner>.highlight>.highlighttable>tbody>tr>.code>div>pre>code,.md-content__inner>.highlight>.highlighttable>tbody>tr>.filename span.filename,.md-content__inner>.highlight>.highlighttable>tbody>tr>.linenos,.md-content__inner>.highlight>pre>code{border-radius:0}.md-content__inner>.highlight+.result{border-left-width:0;border-radius:0;border-right-width:0;margin-left:-.8rem;margin-right:-.8rem}}.md-typeset .keys kbd:after,.md-typeset .keys kbd:before{-moz-osx-font-smoothing:initial;-webkit-font-smoothing:initial;color:inherit;margin:0;position:relative}.md-typeset .keys span{color:var(--md-default-fg-color--light);padding:0 .2em}.md-typeset .keys .key-alt:before,.md-typeset .keys .key-left-alt:before,.md-typeset .keys .key-right-alt:before{content:"⎇";padding-right:.4em}.md-typeset .keys .key-command:before,.md-typeset .keys .key-left-command:before,.md-typeset .keys .key-right-command:before{content:"⌘";padding-right:.4em}.md-typeset .keys .key-control:before,.md-typeset .keys .key-left-control:before,.md-typeset .keys .key-right-control:before{content:"⌃";padding-right:.4em}.md-typeset .keys .key-left-meta:before,.md-typeset .keys .key-meta:before,.md-typeset .keys .key-right-meta:before{content:"◆";padding-right:.4em}.md-typeset .keys .key-left-option:before,.md-typeset .keys .key-option:before,.md-typeset .keys .key-right-option:before{content:"⌥";padding-right:.4em}.md-typeset .keys .key-left-shift:before,.md-typeset .keys .key-right-shift:before,.md-typeset .keys .key-shift:before{content:"⇧";padding-right:.4em}.md-typeset .keys .key-left-super:before,.md-typeset .keys .key-right-super:before,.md-typeset .keys .key-super:before{content:"❖";padding-right:.4em}.md-typeset .keys .key-left-windows:before,.md-typeset .keys .key-right-windows:before,.md-typeset .keys .key-windows:before{content:"⊞";padding-right:.4em}.md-typeset .keys .key-arrow-down:before{content:"↓";padding-right:.4em}.md-typeset .keys .key-arrow-left:before{content:"←";padding-right:.4em}.md-typeset .keys .key-arrow-right:before{content:"→";padding-right:.4em}.md-typeset .keys .key-arrow-up:before{content:"↑";padding-right:.4em}.md-typeset .keys .key-backspace:before{content:"⌫";padding-right:.4em}.md-typeset .keys .key-backtab:before{content:"⇤";padding-right:.4em}.md-typeset .keys .key-caps-lock:before{content:"⇪";padding-right:.4em}.md-typeset .keys .key-clear:before{content:"⌧";padding-right:.4em}.md-typeset .keys .key-context-menu:before{content:"☰";padding-right:.4em}.md-typeset .keys .key-delete:before{content:"⌦";padding-right:.4em}.md-typeset .keys .key-eject:before{content:"⏏";padding-right:.4em}.md-typeset .keys .key-end:before{content:"⤓";padding-right:.4em}.md-typeset .keys .key-escape:before{content:"⎋";padding-right:.4em}.md-typeset .keys .key-home:before{content:"⤒";padding-right:.4em}.md-typeset .keys .key-insert:before{content:"⎀";padding-right:.4em}.md-typeset .keys .key-page-down:before{content:"⇟";padding-right:.4em}.md-typeset .keys .key-page-up:before{content:"⇞";padding-right:.4em}.md-typeset .keys .key-print-screen:before{content:"⎙";padding-right:.4em}.md-typeset .keys .key-tab:after{content:"⇥";padding-left:.4em}.md-typeset .keys .key-num-enter:after{content:"⌤";padding-left:.4em}.md-typeset .keys .key-enter:after{content:"⏎";padding-left:.4em}:root{--md-tabbed-icon--prev:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M15.41 16.58 10.83 12l4.58-4.59L14 6l-6 6 6 6 1.41-1.42Z"/></svg>');--md-tabbed-icon--next:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M8.59 16.58 13.17 12 8.59 7.41 10 6l6 6-6 6-1.41-1.42Z"/></svg>')}.md-typeset .tabbed-set{border-radius:.1rem;display:flex;flex-flow:column wrap;margin:1em 0;position:relative}.md-typeset .tabbed-set>input{height:0;opacity:0;position:absolute;width:0}.md-typeset .tabbed-set>input:target{--md-scroll-offset:0.625em}.md-typeset .tabbed-set>input.focus-visible~.tabbed-labels:before{background-color:var(--md-accent-fg-color)}.md-typeset .tabbed-labels{-ms-overflow-style:none;box-shadow:0 -.05rem var(--md-default-fg-color--lightest) inset;display:flex;max-width:100%;overflow:auto;scrollbar-width:none}@media print{.md-typeset .tabbed-labels{display:contents}}@media screen{.js .md-typeset .tabbed-labels{position:relative}.js .md-typeset .tabbed-labels:before{background:var(--md-default-fg-color);bottom:0;content:"";display:block;height:2px;left:0;position:absolute;transform:translateX(var(--md-indicator-x));transition:width 225ms,background-color .25s,transform .25s;transition-timing-function:cubic-bezier(.4,0,.2,1);width:var(--md-indicator-width)}}.md-typeset .tabbed-labels::-webkit-scrollbar{display:none}.md-typeset .tabbed-labels>label{border-bottom:.1rem solid #0000;border-radius:.1rem .1rem 0 0;color:var(--md-default-fg-color--light);cursor:pointer;flex-shrink:0;font-size:.64rem;font-weight:700;padding:.78125em 1.25em .625em;scroll-margin-inline-start:1rem;transition:background-color .25s,color .25s;white-space:nowrap;width:auto}@media print{.md-typeset .tabbed-labels>label:first-child{order:1}.md-typeset .tabbed-labels>label:nth-child(2){order:2}.md-typeset .tabbed-labels>label:nth-child(3){order:3}.md-typeset .tabbed-labels>label:nth-child(4){order:4}.md-typeset .tabbed-labels>label:nth-child(5){order:5}.md-typeset .tabbed-labels>label:nth-child(6){order:6}.md-typeset .tabbed-labels>label:nth-child(7){order:7}.md-typeset .tabbed-labels>label:nth-child(8){order:8}.md-typeset .tabbed-labels>label:nth-child(9){order:9}.md-typeset .tabbed-labels>label:nth-child(10){order:10}.md-typeset .tabbed-labels>label:nth-child(11){order:11}.md-typeset .tabbed-labels>label:nth-child(12){order:12}.md-typeset .tabbed-labels>label:nth-child(13){order:13}.md-typeset .tabbed-labels>label:nth-child(14){order:14}.md-typeset .tabbed-labels>label:nth-child(15){order:15}.md-typeset .tabbed-labels>label:nth-child(16){order:16}.md-typeset .tabbed-labels>label:nth-child(17){order:17}.md-typeset .tabbed-labels>label:nth-child(18){order:18}.md-typeset .tabbed-labels>label:nth-child(19){order:19}.md-typeset .tabbed-labels>label:nth-child(20){order:20}}.md-typeset .tabbed-labels>label:hover{color:var(--md-default-fg-color)}.md-typeset .tabbed-labels>label>[href]:first-child{color:inherit}.md-typeset .tabbed-labels--linked>label{padding:0}.md-typeset .tabbed-labels--linked>label>a{display:block;padding:.78125em 1.25em .625em}.md-typeset .tabbed-content{width:100%}@media print{.md-typeset .tabbed-content{display:contents}}.md-typeset .tabbed-block{display:none}@media print{.md-typeset .tabbed-block{display:block}.md-typeset .tabbed-block:first-child{order:1}.md-typeset .tabbed-block:nth-child(2){order:2}.md-typeset .tabbed-block:nth-child(3){order:3}.md-typeset .tabbed-block:nth-child(4){order:4}.md-typeset .tabbed-block:nth-child(5){order:5}.md-typeset .tabbed-block:nth-child(6){order:6}.md-typeset .tabbed-block:nth-child(7){order:7}.md-typeset .tabbed-block:nth-child(8){order:8}.md-typeset .tabbed-block:nth-child(9){order:9}.md-typeset .tabbed-block:nth-child(10){order:10}.md-typeset .tabbed-block:nth-child(11){order:11}.md-typeset .tabbed-block:nth-child(12){order:12}.md-typeset .tabbed-block:nth-child(13){order:13}.md-typeset .tabbed-block:nth-child(14){order:14}.md-typeset .tabbed-block:nth-child(15){order:15}.md-typeset .tabbed-block:nth-child(16){order:16}.md-typeset .tabbed-block:nth-child(17){order:17}.md-typeset .tabbed-block:nth-child(18){order:18}.md-typeset .tabbed-block:nth-child(19){order:19}.md-typeset .tabbed-block:nth-child(20){order:20}}.md-typeset .tabbed-block>.highlight:first-child>pre,.md-typeset .tabbed-block>pre:first-child{margin:0}.md-typeset .tabbed-block>.highlight:first-child>pre>code,.md-typeset .tabbed-block>pre:first-child>code{border-top-left-radius:0;border-top-right-radius:0}.md-typeset .tabbed-block>.highlight:first-child>.filename{border-top-left-radius:0;border-top-right-radius:0;margin:0}.md-typeset .tabbed-block>.highlight:first-child>.highlighttable{margin:0}.md-typeset .tabbed-block>.highlight:first-child>.highlighttable>tbody>tr>.filename span.filename,.md-typeset .tabbed-block>.highlight:first-child>.highlighttable>tbody>tr>.linenos{border-top-left-radius:0;border-top-right-radius:0;margin:0}.md-typeset .tabbed-block>.highlight:first-child>.highlighttable>tbody>tr>.code>div>pre>code{border-top-left-radius:0;border-top-right-radius:0}.md-typeset .tabbed-block>.highlight:first-child+.result{margin-top:-.125em}.md-typeset .tabbed-block>.tabbed-set{margin:0}.md-typeset .tabbed-button{align-self:center;border-radius:100%;color:var(--md-default-fg-color--light);cursor:pointer;display:block;height:.9rem;margin-top:.1rem;pointer-events:auto;transition:background-color .25s;width:.9rem}.md-typeset .tabbed-button:hover{background-color:var(--md-accent-fg-color--transparent);color:var(--md-accent-fg-color)}.md-typeset .tabbed-button:after{background-color:currentcolor;content:"";display:block;height:100%;-webkit-mask-image:var(--md-tabbed-icon--prev);mask-image:var(--md-tabbed-icon--prev);-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;transition:background-color .25s,transform .25s;width:100%}.md-typeset .tabbed-control{background:linear-gradient(to right,var(--md-default-bg-color) 60%,#0000);display:flex;height:1.9rem;justify-content:start;pointer-events:none;position:absolute;transition:opacity 125ms;width:1.2rem}[dir=rtl] .md-typeset .tabbed-control{transform:rotate(180deg)}.md-typeset .tabbed-control[hidden]{opacity:0}.md-typeset .tabbed-control--next{background:linear-gradient(to left,var(--md-default-bg-color) 60%,#0000);justify-content:end;right:0}.md-typeset .tabbed-control--next .tabbed-button:after{-webkit-mask-image:var(--md-tabbed-icon--next);mask-image:var(--md-tabbed-icon--next)}@media screen and (max-width:44.984375em){[dir=ltr] .md-content__inner>.tabbed-set .tabbed-labels{padding-left:.8rem}[dir=rtl] .md-content__inner>.tabbed-set .tabbed-labels{padding-right:.8rem}.md-content__inner>.tabbed-set .tabbed-labels{margin:0 -.8rem;max-width:100vw;scroll-padding-inline-start:.8rem}[dir=ltr] .md-content__inner>.tabbed-set .tabbed-labels:after{padding-right:.8rem}[dir=rtl] .md-content__inner>.tabbed-set .tabbed-labels:after{padding-left:.8rem}.md-content__inner>.tabbed-set .tabbed-labels:after{content:""}[dir=ltr] .md-content__inner>.tabbed-set .tabbed-labels~.tabbed-control--prev{padding-left:.8rem}[dir=rtl] .md-content__inner>.tabbed-set .tabbed-labels~.tabbed-control--prev{padding-right:.8rem}[dir=ltr] .md-content__inner>.tabbed-set .tabbed-labels~.tabbed-control--prev{margin-left:-.8rem}[dir=rtl] .md-content__inner>.tabbed-set .tabbed-labels~.tabbed-control--prev{margin-right:-.8rem}.md-content__inner>.tabbed-set .tabbed-labels~.tabbed-control--prev{width:2rem}[dir=ltr] .md-content__inner>.tabbed-set .tabbed-labels~.tabbed-control--next{padding-right:.8rem}[dir=rtl] .md-content__inner>.tabbed-set .tabbed-labels~.tabbed-control--next{padding-left:.8rem}[dir=ltr] .md-content__inner>.tabbed-set .tabbed-labels~.tabbed-control--next{margin-right:-.8rem}[dir=rtl] .md-content__inner>.tabbed-set .tabbed-labels~.tabbed-control--next{margin-left:-.8rem}.md-content__inner>.tabbed-set .tabbed-labels~.tabbed-control--next{width:2rem}}@media screen{.md-typeset .tabbed-set>input:first-child:checked~.tabbed-labels>:first-child,.md-typeset .tabbed-set>input:nth-child(10):checked~.tabbed-labels>:nth-child(10),.md-typeset .tabbed-set>input:nth-child(11):checked~.tabbed-labels>:nth-child(11),.md-typeset .tabbed-set>input:nth-child(12):checked~.tabbed-labels>:nth-child(12),.md-typeset .tabbed-set>input:nth-child(13):checked~.tabbed-labels>:nth-child(13),.md-typeset .tabbed-set>input:nth-child(14):checked~.tabbed-labels>:nth-child(14),.md-typeset .tabbed-set>input:nth-child(15):checked~.tabbed-labels>:nth-child(15),.md-typeset .tabbed-set>input:nth-child(16):checked~.tabbed-labels>:nth-child(16),.md-typeset .tabbed-set>input:nth-child(17):checked~.tabbed-labels>:nth-child(17),.md-typeset .tabbed-set>input:nth-child(18):checked~.tabbed-labels>:nth-child(18),.md-typeset .tabbed-set>input:nth-child(19):checked~.tabbed-labels>:nth-child(19),.md-typeset .tabbed-set>input:nth-child(2):checked~.tabbed-labels>:nth-child(2),.md-typeset .tabbed-set>input:nth-child(20):checked~.tabbed-labels>:nth-child(20),.md-typeset .tabbed-set>input:nth-child(3):checked~.tabbed-labels>:nth-child(3),.md-typeset .tabbed-set>input:nth-child(4):checked~.tabbed-labels>:nth-child(4),.md-typeset .tabbed-set>input:nth-child(5):checked~.tabbed-labels>:nth-child(5),.md-typeset .tabbed-set>input:nth-child(6):checked~.tabbed-labels>:nth-child(6),.md-typeset .tabbed-set>input:nth-child(7):checked~.tabbed-labels>:nth-child(7),.md-typeset .tabbed-set>input:nth-child(8):checked~.tabbed-labels>:nth-child(8),.md-typeset .tabbed-set>input:nth-child(9):checked~.tabbed-labels>:nth-child(9){color:var(--md-default-fg-color)}.md-typeset .no-js .tabbed-set>input:first-child:checked~.tabbed-labels>:first-child,.md-typeset .no-js .tabbed-set>input:nth-child(10):checked~.tabbed-labels>:nth-child(10),.md-typeset .no-js .tabbed-set>input:nth-child(11):checked~.tabbed-labels>:nth-child(11),.md-typeset .no-js .tabbed-set>input:nth-child(12):checked~.tabbed-labels>:nth-child(12),.md-typeset .no-js .tabbed-set>input:nth-child(13):checked~.tabbed-labels>:nth-child(13),.md-typeset .no-js .tabbed-set>input:nth-child(14):checked~.tabbed-labels>:nth-child(14),.md-typeset .no-js .tabbed-set>input:nth-child(15):checked~.tabbed-labels>:nth-child(15),.md-typeset .no-js .tabbed-set>input:nth-child(16):checked~.tabbed-labels>:nth-child(16),.md-typeset .no-js .tabbed-set>input:nth-child(17):checked~.tabbed-labels>:nth-child(17),.md-typeset .no-js .tabbed-set>input:nth-child(18):checked~.tabbed-labels>:nth-child(18),.md-typeset .no-js .tabbed-set>input:nth-child(19):checked~.tabbed-labels>:nth-child(19),.md-typeset .no-js .tabbed-set>input:nth-child(2):checked~.tabbed-labels>:nth-child(2),.md-typeset .no-js .tabbed-set>input:nth-child(20):checked~.tabbed-labels>:nth-child(20),.md-typeset .no-js .tabbed-set>input:nth-child(3):checked~.tabbed-labels>:nth-child(3),.md-typeset .no-js .tabbed-set>input:nth-child(4):checked~.tabbed-labels>:nth-child(4),.md-typeset .no-js .tabbed-set>input:nth-child(5):checked~.tabbed-labels>:nth-child(5),.md-typeset .no-js .tabbed-set>input:nth-child(6):checked~.tabbed-labels>:nth-child(6),.md-typeset .no-js .tabbed-set>input:nth-child(7):checked~.tabbed-labels>:nth-child(7),.md-typeset .no-js .tabbed-set>input:nth-child(8):checked~.tabbed-labels>:nth-child(8),.md-typeset .no-js .tabbed-set>input:nth-child(9):checked~.tabbed-labels>:nth-child(9),.no-js .md-typeset .tabbed-set>input:first-child:checked~.tabbed-labels>:first-child,.no-js .md-typeset .tabbed-set>input:nth-child(10):checked~.tabbed-labels>:nth-child(10),.no-js .md-typeset .tabbed-set>input:nth-child(11):checked~.tabbed-labels>:nth-child(11),.no-js .md-typeset .tabbed-set>input:nth-child(12):checked~.tabbed-labels>:nth-child(12),.no-js .md-typeset .tabbed-set>input:nth-child(13):checked~.tabbed-labels>:nth-child(13),.no-js .md-typeset .tabbed-set>input:nth-child(14):checked~.tabbed-labels>:nth-child(14),.no-js .md-typeset .tabbed-set>input:nth-child(15):checked~.tabbed-labels>:nth-child(15),.no-js .md-typeset .tabbed-set>input:nth-child(16):checked~.tabbed-labels>:nth-child(16),.no-js .md-typeset .tabbed-set>input:nth-child(17):checked~.tabbed-labels>:nth-child(17),.no-js .md-typeset .tabbed-set>input:nth-child(18):checked~.tabbed-labels>:nth-child(18),.no-js .md-typeset .tabbed-set>input:nth-child(19):checked~.tabbed-labels>:nth-child(19),.no-js .md-typeset .tabbed-set>input:nth-child(2):checked~.tabbed-labels>:nth-child(2),.no-js .md-typeset .tabbed-set>input:nth-child(20):checked~.tabbed-labels>:nth-child(20),.no-js .md-typeset .tabbed-set>input:nth-child(3):checked~.tabbed-labels>:nth-child(3),.no-js .md-typeset .tabbed-set>input:nth-child(4):checked~.tabbed-labels>:nth-child(4),.no-js .md-typeset .tabbed-set>input:nth-child(5):checked~.tabbed-labels>:nth-child(5),.no-js .md-typeset .tabbed-set>input:nth-child(6):checked~.tabbed-labels>:nth-child(6),.no-js .md-typeset .tabbed-set>input:nth-child(7):checked~.tabbed-labels>:nth-child(7),.no-js .md-typeset .tabbed-set>input:nth-child(8):checked~.tabbed-labels>:nth-child(8),.no-js .md-typeset .tabbed-set>input:nth-child(9):checked~.tabbed-labels>:nth-child(9){border-color:var(--md-default-fg-color)}}.md-typeset .tabbed-set>input:first-child.focus-visible~.tabbed-labels>:first-child,.md-typeset .tabbed-set>input:nth-child(10).focus-visible~.tabbed-labels>:nth-child(10),.md-typeset .tabbed-set>input:nth-child(11).focus-visible~.tabbed-labels>:nth-child(11),.md-typeset .tabbed-set>input:nth-child(12).focus-visible~.tabbed-labels>:nth-child(12),.md-typeset .tabbed-set>input:nth-child(13).focus-visible~.tabbed-labels>:nth-child(13),.md-typeset .tabbed-set>input:nth-child(14).focus-visible~.tabbed-labels>:nth-child(14),.md-typeset .tabbed-set>input:nth-child(15).focus-visible~.tabbed-labels>:nth-child(15),.md-typeset .tabbed-set>input:nth-child(16).focus-visible~.tabbed-labels>:nth-child(16),.md-typeset .tabbed-set>input:nth-child(17).focus-visible~.tabbed-labels>:nth-child(17),.md-typeset .tabbed-set>input:nth-child(18).focus-visible~.tabbed-labels>:nth-child(18),.md-typeset .tabbed-set>input:nth-child(19).focus-visible~.tabbed-labels>:nth-child(19),.md-typeset .tabbed-set>input:nth-child(2).focus-visible~.tabbed-labels>:nth-child(2),.md-typeset .tabbed-set>input:nth-child(20).focus-visible~.tabbed-labels>:nth-child(20),.md-typeset .tabbed-set>input:nth-child(3).focus-visible~.tabbed-labels>:nth-child(3),.md-typeset .tabbed-set>input:nth-child(4).focus-visible~.tabbed-labels>:nth-child(4),.md-typeset .tabbed-set>input:nth-child(5).focus-visible~.tabbed-labels>:nth-child(5),.md-typeset .tabbed-set>input:nth-child(6).focus-visible~.tabbed-labels>:nth-child(6),.md-typeset .tabbed-set>input:nth-child(7).focus-visible~.tabbed-labels>:nth-child(7),.md-typeset .tabbed-set>input:nth-child(8).focus-visible~.tabbed-labels>:nth-child(8),.md-typeset .tabbed-set>input:nth-child(9).focus-visible~.tabbed-labels>:nth-child(9){color:var(--md-accent-fg-color)}.md-typeset .tabbed-set>input:first-child:checked~.tabbed-content>:first-child,.md-typeset .tabbed-set>input:nth-child(10):checked~.tabbed-content>:nth-child(10),.md-typeset .tabbed-set>input:nth-child(11):checked~.tabbed-content>:nth-child(11),.md-typeset .tabbed-set>input:nth-child(12):checked~.tabbed-content>:nth-child(12),.md-typeset .tabbed-set>input:nth-child(13):checked~.tabbed-content>:nth-child(13),.md-typeset .tabbed-set>input:nth-child(14):checked~.tabbed-content>:nth-child(14),.md-typeset .tabbed-set>input:nth-child(15):checked~.tabbed-content>:nth-child(15),.md-typeset .tabbed-set>input:nth-child(16):checked~.tabbed-content>:nth-child(16),.md-typeset .tabbed-set>input:nth-child(17):checked~.tabbed-content>:nth-child(17),.md-typeset .tabbed-set>input:nth-child(18):checked~.tabbed-content>:nth-child(18),.md-typeset .tabbed-set>input:nth-child(19):checked~.tabbed-content>:nth-child(19),.md-typeset .tabbed-set>input:nth-child(2):checked~.tabbed-content>:nth-child(2),.md-typeset .tabbed-set>input:nth-child(20):checked~.tabbed-content>:nth-child(20),.md-typeset .tabbed-set>input:nth-child(3):checked~.tabbed-content>:nth-child(3),.md-typeset .tabbed-set>input:nth-child(4):checked~.tabbed-content>:nth-child(4),.md-typeset .tabbed-set>input:nth-child(5):checked~.tabbed-content>:nth-child(5),.md-typeset .tabbed-set>input:nth-child(6):checked~.tabbed-content>:nth-child(6),.md-typeset .tabbed-set>input:nth-child(7):checked~.tabbed-content>:nth-child(7),.md-typeset .tabbed-set>input:nth-child(8):checked~.tabbed-content>:nth-child(8),.md-typeset .tabbed-set>input:nth-child(9):checked~.tabbed-content>:nth-child(9){display:block}:root{--md-tasklist-icon:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M1 12C1 5.925 5.925 1 12 1s11 4.925 11 11-4.925 11-11 11S1 18.075 1 12Zm16.28-2.72a.751.751 0 0 0-.018-1.042.751.751 0 0 0-1.042-.018l-5.97 5.97-2.47-2.47a.751.751 0 0 0-1.042.018.751.751 0 0 0-.018 1.042l3 3a.75.75 0 0 0 1.06 0Z"/></svg>');--md-tasklist-icon--checked:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M1 12C1 5.925 5.925 1 12 1s11 4.925 11 11-4.925 11-11 11S1 18.075 1 12Zm16.28-2.72a.751.751 0 0 0-.018-1.042.751.751 0 0 0-1.042-.018l-5.97 5.97-2.47-2.47a.751.751 0 0 0-1.042.018.751.751 0 0 0-.018 1.042l3 3a.75.75 0 0 0 1.06 0Z"/></svg>')}.md-typeset .task-list-item{list-style-type:none;position:relative}[dir=ltr] .md-typeset .task-list-item [type=checkbox]{left:-2em}[dir=rtl] .md-typeset .task-list-item [type=checkbox]{right:-2em}.md-typeset .task-list-item [type=checkbox]{position:absolute;top:.45em}.md-typeset .task-list-control [type=checkbox]{opacity:0;z-index:-1}[dir=ltr] .md-typeset .task-list-indicator:before{left:-1.5em}[dir=rtl] .md-typeset .task-list-indicator:before{right:-1.5em}.md-typeset .task-list-indicator:before{background-color:var(--md-default-fg-color--lightest);content:"";height:1.25em;-webkit-mask-image:var(--md-tasklist-icon);mask-image:var(--md-tasklist-icon);-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;position:absolute;top:.15em;width:1.25em}.md-typeset [type=checkbox]:checked+.task-list-indicator:before{background-color:#00e676;-webkit-mask-image:var(--md-tasklist-icon--checked);mask-image:var(--md-tasklist-icon--checked)}:root>*{--md-mermaid-font-family:var(--md-text-font-family),sans-serif;--md-mermaid-edge-color:var(--md-code-fg-color);--md-mermaid-node-bg-color:var(--md-accent-fg-color--transparent);--md-mermaid-node-fg-color:var(--md-accent-fg-color);--md-mermaid-label-bg-color:var(--md-default-bg-color);--md-mermaid-label-fg-color:var(--md-code-fg-color);--md-mermaid-sequence-actor-bg-color:var(--md-mermaid-label-bg-color);--md-mermaid-sequence-actor-fg-color:var(--md-mermaid-label-fg-color);--md-mermaid-sequence-actor-border-color:var(--md-mermaid-node-fg-color);--md-mermaid-sequence-actor-line-color:var(--md-default-fg-color--lighter);--md-mermaid-sequence-actorman-bg-color:var(--md-mermaid-label-bg-color);--md-mermaid-sequence-actorman-line-color:var(--md-mermaid-node-fg-color);--md-mermaid-sequence-box-bg-color:var(--md-mermaid-node-bg-color);--md-mermaid-sequence-box-fg-color:var(--md-mermaid-edge-color);--md-mermaid-sequence-label-bg-color:var(--md-mermaid-node-bg-color);--md-mermaid-sequence-label-fg-color:var(--md-mermaid-node-fg-color);--md-mermaid-sequence-loop-bg-color:var(--md-mermaid-node-bg-color);--md-mermaid-sequence-loop-fg-color:var(--md-mermaid-edge-color);--md-mermaid-sequence-loop-border-color:var(--md-mermaid-node-fg-color);--md-mermaid-sequence-message-fg-color:var(--md-mermaid-edge-color);--md-mermaid-sequence-message-line-color:var(--md-mermaid-edge-color);--md-mermaid-sequence-note-bg-color:var(--md-mermaid-label-bg-color);--md-mermaid-sequence-note-fg-color:var(--md-mermaid-edge-color);--md-mermaid-sequence-note-border-color:var(--md-mermaid-label-fg-color);--md-mermaid-sequence-number-bg-color:var(--md-mermaid-node-fg-color);--md-mermaid-sequence-number-fg-color:var(--md-accent-bg-color)}.mermaid{line-height:normal;margin:1em 0}.md-typeset .grid{grid-gap:.4rem;display:grid;grid-template-columns:repeat(auto-fit,minmax(min(100%,16rem),1fr));margin:1em 0}.md-typeset .grid.cards>ol,.md-typeset .grid.cards>ul{display:contents}.md-typeset .grid.cards>ol>li,.md-typeset .grid.cards>ul>li,.md-typeset .grid>.card{border:.05rem solid var(--md-default-fg-color--lightest);border-radius:.1rem;display:block;margin:0;padding:.8rem;transition:border .25s,box-shadow .25s}.md-typeset .grid.cards>ol>li:focus-within,.md-typeset .grid.cards>ol>li:hover,.md-typeset .grid.cards>ul>li:focus-within,.md-typeset .grid.cards>ul>li:hover,.md-typeset .grid>.card:focus-within,.md-typeset .grid>.card:hover{border-color:#0000;box-shadow:var(--md-shadow-z2)}.md-typeset .grid.cards>ol>li>hr,.md-typeset .grid.cards>ul>li>hr,.md-typeset .grid>.card>hr{margin-bottom:1em;margin-top:1em}.md-typeset .grid.cards>ol>li>:first-child,.md-typeset .grid.cards>ul>li>:first-child,.md-typeset .grid>.card>:first-child{margin-top:0}.md-typeset .grid.cards>ol>li>:last-child,.md-typeset .grid.cards>ul>li>:last-child,.md-typeset .grid>.card>:last-child{margin-bottom:0}.md-typeset .grid>*,.md-typeset .grid>.admonition,.md-typeset .grid>.highlight>*,.md-typeset .grid>.highlighttable,.md-typeset .grid>.md-typeset details,.md-typeset .grid>details,.md-typeset .grid>pre{margin-bottom:0;margin-top:0}.md-typeset .grid>.highlight>pre:only-child,.md-typeset .grid>.highlight>pre>code,.md-typeset .grid>.highlighttable,.md-typeset .grid>.highlighttable>tbody,.md-typeset .grid>.highlighttable>tbody>tr,.md-typeset .grid>.highlighttable>tbody>tr>.code,.md-typeset .grid>.highlighttable>tbody>tr>.code>.highlight,.md-typeset .grid>.highlighttable>tbody>tr>.code>.highlight>pre,.md-typeset .grid>.highlighttable>tbody>tr>.code>.highlight>pre>code{height:100%}.md-typeset .grid>.tabbed-set{margin-bottom:0;margin-top:0}@media screen and (min-width:45em){[dir=ltr] .md-typeset .inline{float:left}[dir=rtl] .md-typeset .inline{float:right}[dir=ltr] .md-typeset .inline{margin-right:.8rem}[dir=rtl] .md-typeset .inline{margin-left:.8rem}.md-typeset .inline{margin-bottom:.8rem;margin-top:0;width:11.7rem}[dir=ltr] .md-typeset .inline.end{float:right}[dir=rtl] .md-typeset .inline.end{float:left}[dir=ltr] .md-typeset .inline.end{margin-left:.8rem;margin-right:0}[dir=rtl] .md-typeset .inline.end{margin-left:0;margin-right:.8rem}}
\ No newline at end of file
diff --git a/assets/stylesheets/main.7e359304.min.css.map b/assets/stylesheets/main.7e359304.min.css.map
new file mode 100644
index 000000000..15b8fcf1e
--- /dev/null
+++ b/assets/stylesheets/main.7e359304.min.css.map
@@ -0,0 +1 @@
+{"version":3,"sources":["src/templates/assets/stylesheets/main/components/_meta.scss","../../../../src/templates/assets/stylesheets/main.scss","src/templates/assets/stylesheets/main/_resets.scss","src/templates/assets/stylesheets/main/_colors.scss","src/templates/assets/stylesheets/main/_icons.scss","src/templates/assets/stylesheets/main/_typeset.scss","src/templates/assets/stylesheets/utilities/_break.scss","src/templates/assets/stylesheets/main/components/_author.scss","src/templates/assets/stylesheets/main/components/_banner.scss","src/templates/assets/stylesheets/main/components/_base.scss","src/templates/assets/stylesheets/main/components/_clipboard.scss","src/templates/assets/stylesheets/main/components/_consent.scss","src/templates/assets/stylesheets/main/components/_content.scss","src/templates/assets/stylesheets/main/components/_dialog.scss","src/templates/assets/stylesheets/main/components/_feedback.scss","src/templates/assets/stylesheets/main/components/_footer.scss","src/templates/assets/stylesheets/main/components/_form.scss","src/templates/assets/stylesheets/main/components/_header.scss","node_modules/material-design-color/material-color.scss","src/templates/assets/stylesheets/main/components/_nav.scss","src/templates/assets/stylesheets/main/components/_pagination.scss","src/templates/assets/stylesheets/main/components/_post.scss","src/templates/assets/stylesheets/main/components/_progress.scss","src/templates/assets/stylesheets/main/components/_search.scss","src/templates/assets/stylesheets/main/components/_select.scss","src/templates/assets/stylesheets/main/components/_sidebar.scss","src/templates/assets/stylesheets/main/components/_source.scss","src/templates/assets/stylesheets/main/components/_status.scss","src/templates/assets/stylesheets/main/components/_tabs.scss","src/templates/assets/stylesheets/main/components/_tag.scss","src/templates/assets/stylesheets/main/components/_tooltip.scss","src/templates/assets/stylesheets/main/components/_top.scss","src/templates/assets/stylesheets/main/components/_version.scss","src/templates/assets/stylesheets/main/extensions/markdown/_admonition.scss","src/templates/assets/stylesheets/main/extensions/markdown/_footnotes.scss","src/templates/assets/stylesheets/main/extensions/markdown/_toc.scss","src/templates/assets/stylesheets/main/extensions/pymdownx/_arithmatex.scss","src/templates/assets/stylesheets/main/extensions/pymdownx/_critic.scss","src/templates/assets/stylesheets/main/extensions/pymdownx/_details.scss","src/templates/assets/stylesheets/main/extensions/pymdownx/_emoji.scss","src/templates/assets/stylesheets/main/extensions/pymdownx/_highlight.scss","src/templates/assets/stylesheets/main/extensions/pymdownx/_keys.scss","src/templates/assets/stylesheets/main/extensions/pymdownx/_tabbed.scss","src/templates/assets/stylesheets/main/extensions/pymdownx/_tasklist.scss","src/templates/assets/stylesheets/main/integrations/_mermaid.scss","src/templates/assets/stylesheets/main/modifiers/_grid.scss","src/templates/assets/stylesheets/main/modifiers/_inline.scss"],"names":[],"mappings":"AA0CE,gBCgxCF,CC9xCA,KAEE,6BAAA,CAAA,0BAAA,CAAA,qBAAA,CADA,qBDzBF,CC8BA,iBAGE,kBD3BF,CC8BE,gCANF,iBAOI,yBDzBF,CACF,CC6BA,KACE,QD1BF,CC8BA,qBAIE,uCD3BF,CC+BA,EACE,aAAA,CACA,oBD5BF,CCgCA,GAME,QAAA,CALA,kBAAA,CACA,aAAA,CACA,aAAA,CAEA,gBAAA,CADA,SD3BF,CCiCA,MACE,aD9BF,CCkCA,QAEE,eD/BF,CCmCA,IACE,iBDhCF,CCoCA,MAEE,uBAAA,CADA,gBDhCF,CCqCA,MAEE,eAAA,CACA,kBDlCF,CCsCA,OAKE,gBAAA,CACA,QAAA,CAHA,mBAAA,CACA,iBAAA,CAFA,QAAA,CADA,SD9BF,CCuCA,MACE,QAAA,CACA,YDpCF,CErDA,MAIE,6BAAA,CACA,oCAAA,CACA,mCAAA,CACA,0BAAA,CACA,sCAAA,CAGA,4BAAA,CACA,2CAAA,CACA,yBAAA,CACA,qCFmDF,CE7CA,+BAIE,kBF6CF,CE1CE,oHAEE,YF4CJ,CEnCA,qCAIE,eAAA,CAGA,+BAAA,CACA,sCAAA,CACA,wCAAA,CACA,yCAAA,CACA,0BAAA,CACA,sCAAA,CACA,wCAAA,CACA,yCAAA,CAGA,0BAAA,CACA,0BAAA,CAGA,0BAAA,CACA,mCAAA,CAGA,iCAAA,CACA,kCAAA,CACA,mCAAA,CACA,mCAAA,CACA,kCAAA,CACA,iCAAA,CACA,+CAAA,CACA,6DAAA,CACA,gEAAA,CACA,4DAAA,CACA,4DAAA,CACA,6DAAA,CAGA,6CAAA,CAGA,+CAAA,CAGA,gCAAA,CACA,gCAAA,CAGA,8BAAA,CACA,kCAAA,CACA,qCAAA,CAGA,iCAAA,CAGA,kCAAA,CACA,gDAAA,CAGA,mDAAA,CACA,mDAAA,CAGA,+BAAA,CACA,0BAAA,CAGA,yBAAA,CACA,qCAAA,CACA,uCAAA,CACA,8BAAA,CACA,oCAAA,CAGA,8DAAA,CAKA,8DAAA,CAKA,0DFKF,CG9HE,aAIE,iBAAA,CAHA,aAAA,CAEA,aAAA,CADA,YHmIJ,CIxIA,KACE,kCAAA,CACA,iCAAA,CAGA,uGAAA,CAKA,mFJyIF,CInIA,iBAIE,mCAAA,CACA,6BAAA,CAFA,sCJwIF,CIlIA,aAIE,4BAAA,CADA,sCJsIF,CI7HA,MACE,0NAAA,CACA,mNAAA,CACA,oNJgIF,CIzHA,YAGE,gCAAA,CAAA,kBAAA,CAFA,eAAA,CACA,eJ6HF,CIxHE,aAPF,YAQI,gBJ2HF,CACF,CIxHE,uGAME,iBAAA,CAAA,cJ0HJ,CItHE,eAKE,uCAAA,CAHA,aAAA,CAEA,eAAA,CAHA,iBJ6HJ,CIpHE,8BAPE,eAAA,CAGA,qBJ+HJ,CI3HE,eAEE,kBAAA,CAEA,eAAA,CAHA,oBJ0HJ,CIlHE,eAEE,gBAAA,CACA,eAAA,CAEA,qBAAA,CADA,eAAA,CAHA,mBJwHJ,CIhHE,kBACE,eJkHJ,CI9GE,eAEE,eAAA,CACA,qBAAA,CAFA,YJkHJ,CI5GE,8BAKE,uCAAA,CAFA,cAAA,CACA,eAAA,CAEA,qBAAA,CAJA,eJkHJ,CI1GE,eACE,wBJ4GJ,CIxGE,eAGE,+DAAA,CAFA,iBAAA,CACA,cJ2GJ,CItGE,cACE,+BAAA,CACA,qBJwGJ,CIrGI,mCAEE,sBJsGN,CIlGI,wCACE,+BJoGN,CIjGM,kDACE,uDJmGR,CI9FI,mBACE,kBAAA,CACA,iCJgGN,CI5FI,4BACE,uCAAA,CACA,oBJ8FN,CIzFE,iDAIE,6BAAA,CACA,aAAA,CAFA,2BJ6FJ,CIxFI,aARF,iDASI,oBJ6FJ,CACF,CIzFE,iBAIE,wCAAA,CACA,mBAAA,CACA,kCAAA,CAAA,0BAAA,CAJA,eAAA,CADA,uBAAA,CAEA,qBJ8FJ,CIxFI,qCAEE,uCAAA,CADA,YJ2FN,CIrFE,gBAEE,iBAAA,CACA,eAAA,CAFA,iBJyFJ,CIpFI,qBASE,kCAAA,CAAA,0BAAA,CADA,eAAA,CAPA,aAAA,CAEA,QAAA,CAIA,uCAAA,CAHA,aAAA,CAFA,oCAAA,CASA,yDAAA,CADA,oBAAA,CAJA,iBAAA,CADA,iBJ4FN,CInFM,2BACE,+CJqFR,CIjFM,wCAEE,YAAA,CADA,WJoFR,CI/EM,8CACE,oDJiFR,CI9EQ,oDACE,0CJgFV,CIzEE,gBAOE,4CAAA,CACA,mBAAA,CACA,mKACE,CANF,gCAAA,CAHA,oBAAA,CAEA,eAAA,CADA,uBAAA,CAIA,uBAAA,CADA,qBJ+EJ,CIpEE,iBAGE,6CAAA,CACA,kCAAA,CAAA,0BAAA,CAHA,aAAA,CACA,qBJwEJ,CIlEE,iBAGE,6DAAA,CADA,WAAA,CADA,oBJsEJ,CIhEE,kBACE,WJkEJ,CI9DE,oDAEE,qBJgEJ,CIlEE,oDAEE,sBJgEJ,CI5DE,iCACE,kBJiEJ,CIlEE,iCACE,mBJiEJ,CIlEE,iCAIE,2DJ8DJ,CIlEE,iCAIE,4DJ8DJ,CIlEE,uBAGE,uCAAA,CADA,aAAA,CAAA,cJgEJ,CI1DE,eACE,oBJ4DJ,CIxDE,kDAGE,kBJ0DJ,CI7DE,kDAGE,mBJ0DJ,CI7DE,8BAEE,SJ2DJ,CIvDI,0DACE,iBJ0DN,CItDI,oCACE,2BJyDN,CItDM,0CACE,2BJyDR,CIpDI,wDACE,kBJwDN,CIzDI,wDACE,mBJwDN,CIzDI,oCAEE,kBJuDN,CIpDM,kGAEE,aJwDR,CIpDM,0DACE,eJuDR,CInDM,4HAEE,kBJsDR,CIxDM,4HAEE,mBJsDR,CIxDM,oFACE,kBAAA,CAAA,eJuDR,CIhDE,yBAEE,mBJkDJ,CIpDE,yBAEE,oBJkDJ,CIpDE,eACE,mBAAA,CAAA,cJmDJ,CI9CE,kDAIE,WAAA,CADA,cJiDJ,CIzCI,4BAEE,oBJ2CN,CIvCI,6BAEE,oBJyCN,CIrCI,kCACE,YJuCN,CIlCE,mBACE,iBAAA,CAGA,eAAA,CADA,cAAA,CAEA,iBAAA,CAHA,yBAAA,CAAA,sBAAA,CAAA,iBJuCJ,CIjCI,uBACE,aAAA,CACA,aJmCN,CI9BE,uBAGE,iBAAA,CADA,eAAA,CADA,eJkCJ,CI5BE,mBACE,cJ8BJ,CI1BE,+BAME,2CAAA,CACA,iDAAA,CACA,mBAAA,CAPA,oBAAA,CAGA,gBAAA,CAFA,cAAA,CACA,aAAA,CAEA,iBJ+BJ,CIzBI,aAXF,+BAYI,aJ4BJ,CACF,CIvBI,iCACE,gBJyBN,CIlBM,8FACE,YJoBR,CIhBM,4FACE,eJkBR,CIbI,8FACE,eJeN,CIZM,kHACE,gBJcR,CITI,kCAGE,eAAA,CAFA,cAAA,CACA,sBAAA,CAEA,kBJWN,CIPI,kCAGE,qDAAA,CAFA,sBAAA,CACA,kBJUN,CILI,wCACE,iCJON,CIJM,8CACE,qDAAA,CACA,sDJMR,CIDI,iCACE,iBJGN,CIEE,wCACE,cJAJ,CIGI,wDAIE,gBJKN,CITI,wDAIE,iBJKN,CITI,8CAME,UAAA,CALA,oBAAA,CAEA,YAAA,CAKA,oDAAA,CAAA,4CAAA,CACA,6BAAA,CAAA,qBAAA,CACA,yBAAA,CAAA,iBAAA,CAHA,iCAAA,CAFA,0BAAA,CAHA,WJON,CIKI,oDACE,oDJHN,CIOI,mEACE,kDAAA,CACA,yDAAA,CAAA,iDJLN,CISI,oEACE,kDAAA,CACA,0DAAA,CAAA,kDJPN,CIYE,wBACE,iBAAA,CACA,eAAA,CACA,iBJVJ,CIcE,mBACE,oBAAA,CAEA,kBAAA,CADA,eJXJ,CIeI,aANF,mBAOI,aJZJ,CACF,CIeI,8BACE,aAAA,CAEA,QAAA,CACA,eAAA,CAFA,UJXN,CKnVI,0CD6WF,uBACE,iBJtBF,CIyBE,4BACE,eJvBJ,CACF,CMlhBE,uBAOE,kBAAA,CALA,aAAA,CACA,aAAA,CAEA,aAAA,CACA,eAAA,CALA,iBAAA,CAOA,sCACE,CALF,YNwhBJ,CM/gBI,2BACE,aNihBN,CM7gBI,6BAME,+CAAA,CAFA,yCAAA,CAHA,eAAA,CACA,eAAA,CACA,kBAAA,CAEA,iBNghBN,CM3gBI,6BAEE,aAAA,CADA,YN8gBN,CMxgBE,wBACE,kBN0gBJ,CMvgBI,4BACE,mCAAA,CACA,uBNygBN,CMrgBI,4DAEE,oBAAA,CADA,SNwgBN,CMpgBM,oEACE,mBNsgBR,CO5jBA,WAGE,0CAAA,CADA,+BAAA,CADA,aPikBF,CO5jBE,aANF,WAOI,YP+jBF,CACF,CO5jBE,oBAEE,2CAAA,CADA,gCP+jBJ,CO1jBE,kBAGE,eAAA,CADA,iBAAA,CADA,eP8jBJ,COxjBE,6BACE,WP6jBJ,CO9jBE,6BACE,UP6jBJ,CO9jBE,mBAEE,aAAA,CACA,cAAA,CACA,uBP0jBJ,COvjBI,0BACE,YPyjBN,COrjBI,yBACE,UPujBN,CQ5lBA,KASE,cAAA,CARA,WAAA,CACA,iBRgmBF,CK5bI,oCGtKJ,KAaI,gBRylBF,CACF,CKjcI,oCGtKJ,KAkBI,cRylBF,CACF,CQplBA,KASE,2CAAA,CAPA,YAAA,CACA,qBAAA,CAKA,eAAA,CAHA,eAAA,CAJA,iBAAA,CAGA,UR0lBF,CQllBE,aAZF,KAaI,aRqlBF,CACF,CKlcI,0CGhJF,yBAII,cRklBJ,CACF,CQzkBA,SAEE,gBAAA,CAAA,iBAAA,CADA,eR6kBF,CQxkBA,cACE,YAAA,CACA,qBAAA,CACA,WR2kBF,CQxkBE,aANF,cAOI,aR2kBF,CACF,CQvkBA,SACE,WR0kBF,CQvkBE,gBACE,YAAA,CACA,WAAA,CACA,iBRykBJ,CQpkBA,aACE,eAAA,CACA,sBRukBF,CQ9jBA,WACE,YRikBF,CQ5jBA,WAGE,QAAA,CACA,SAAA,CAHA,iBAAA,CACA,ORikBF,CQ5jBE,uCACE,aR8jBJ,CQ1jBE,+BAEE,uCAAA,CADA,kBR6jBJ,CQvjBA,SASE,2CAAA,CACA,mBAAA,CAFA,gCAAA,CADA,gBAAA,CADA,YAAA,CAMA,SAAA,CADA,uCAAA,CANA,mBAAA,CAJA,cAAA,CAYA,2BAAA,CATA,URikBF,CQrjBE,eAEE,SAAA,CAIA,uBAAA,CAHA,oEACE,CAHF,UR0jBJ,CQ5iBA,MACE,WR+iBF,CSxsBA,MACE,+PT0sBF,CSpsBA,cASE,mBAAA,CAFA,0CAAA,CACA,cAAA,CAFA,YAAA,CAIA,uCAAA,CACA,oBAAA,CAVA,iBAAA,CAEA,UAAA,CADA,QAAA,CAUA,qBAAA,CAPA,WAAA,CADA,ST+sBF,CSpsBE,aAfF,cAgBI,YTusBF,CACF,CSpsBE,kCAEE,uCAAA,CADA,YTusBJ,CSlsBE,qBACE,uCTosBJ,CShsBE,wCACE,+BTksBJ,CS7rBE,oBAME,6BAAA,CADA,UAAA,CAJA,aAAA,CAEA,cAAA,CACA,aAAA,CAGA,2CAAA,CAAA,mCAAA,CACA,4BAAA,CAAA,oBAAA,CACA,6BAAA,CAAA,qBAAA,CACA,yBAAA,CAAA,iBAAA,CARA,aTusBJ,CS3rBE,sBACE,cT6rBJ,CS1rBI,2BACE,2CT4rBN,CStrBI,kEAEE,uDAAA,CADA,+BTyrBN,CU/vBA,mBACE,GACE,SAAA,CACA,0BVkwBF,CU/vBA,GACE,SAAA,CACA,uBViwBF,CACF,CU7vBA,mBACE,GACE,SV+vBF,CU5vBA,GACE,SV8vBF,CACF,CUnvBE,qBASE,2BAAA,CADA,mCAAA,CAAA,2BAAA,CAFA,0BAAA,CADA,WAAA,CAEA,SAAA,CANA,cAAA,CACA,KAAA,CAEA,UAAA,CADA,SV2vBJ,CUjvBE,mBAcE,mDAAA,CANA,2CAAA,CACA,QAAA,CACA,mBAAA,CARA,QAAA,CASA,kDACE,CAPF,eAAA,CAEA,aAAA,CADA,SAAA,CALA,cAAA,CAGA,UAAA,CADA,SV4vBJ,CU7uBE,kBACE,aV+uBJ,CU3uBE,sBACE,YAAA,CACA,YV6uBJ,CU1uBI,oCACE,aV4uBN,CUvuBE,sBACE,mBVyuBJ,CUtuBI,6CACE,cVwuBN,CKloBI,0CKvGA,6CAKI,aAAA,CAEA,gBAAA,CACA,iBAAA,CAFA,UV0uBN,CACF,CUnuBE,kBACE,cVquBJ,CWt0BA,YACE,WAAA,CAIA,WXs0BF,CWn0BE,mBAEE,qBAAA,CADA,iBXs0BJ,CKzqBI,sCMtJE,4EACE,kBXk0BN,CW9zBI,0JACE,mBXg0BN,CWj0BI,8EACE,kBXg0BN,CACF,CW3zBI,0BAGE,UAAA,CAFA,aAAA,CACA,YX8zBN,CWzzBI,+BACE,eX2zBN,CWrzBE,8BACE,WX0zBJ,CW3zBE,8BACE,UX0zBJ,CW3zBE,8BAIE,iBXuzBJ,CW3zBE,8BAIE,kBXuzBJ,CW3zBE,oBAGE,cAAA,CADA,SXyzBJ,CWpzBI,aAPF,oBAQI,YXuzBJ,CACF,CWpzBI,gCACE,yCXszBN,CWlzBI,wBACE,cAAA,CACA,kBXozBN,CWjzBM,kCACE,oBXmzBR,CYp3BA,qBAeE,WZq3BF,CYp4BA,qBAeE,UZq3BF,CYp4BA,WAOE,2CAAA,CACA,mBAAA,CANA,YAAA,CAOA,8BAAA,CALA,iBAAA,CAMA,SAAA,CALA,mBAAA,CACA,mBAAA,CALA,cAAA,CAaA,0BAAA,CAHA,wCACE,CATF,SZi4BF,CYl3BE,aAlBF,WAmBI,YZq3BF,CACF,CYl3BE,mBAEE,SAAA,CADA,mBAAA,CAKA,uBAAA,CAHA,kEZq3BJ,CY92BE,kBAEE,gCAAA,CADA,eZi3BJ,Can5BA,aACE,gBAAA,CACA,iBbs5BF,Can5BE,sBAGE,WAAA,CADA,QAAA,CADA,Sbu5BJ,Caj5BE,oBAEE,eAAA,CADA,ebo5BJ,Ca/4BE,oBACE,iBbi5BJ,Ca74BE,mBAEE,YAAA,CACA,cAAA,CACA,6BAAA,CAHA,iBbk5BJ,Ca54BI,iDACE,yCb84BN,Ca14BI,6BACE,iBb44BN,Cav4BE,mBAGE,uCAAA,CACA,cAAA,CAHA,aAAA,CACA,cAAA,CAGA,sBby4BJ,Cat4BI,gDACE,+Bbw4BN,Cap4BI,4BACE,0CAAA,CACA,mBbs4BN,Caj4BE,mBAEE,SAAA,CADA,iBAAA,CAKA,2BAAA,CAHA,8Dbo4BJ,Ca93BI,qBAEE,aAAA,CADA,ebi4BN,Ca53BI,6BACE,SAAA,CACA,uBb83BN,Cc58BA,WAEE,0CAAA,CADA,+Bdg9BF,Cc58BE,aALF,WAMI,Yd+8BF,CACF,Cc58BE,kBACE,6BAAA,CAEA,aAAA,CADA,ad+8BJ,Cc38BI,gCACE,Yd68BN,Ccx8BE,iBAOE,eAAA,CANA,YAAA,CAKA,cAAA,CAGA,mBAAA,CAAA,eAAA,CADA,cAAA,CAGA,uCAAA,CADA,eAAA,CAEA,uBds8BJ,Ccn8BI,8CACE,Udq8BN,Ccj8BI,+BACE,oBdm8BN,CKrzBI,0CSvIE,uBACE,ad+7BN,Cc57BM,yCACE,Yd87BR,CACF,Ccz7BI,iCACE,gBd47BN,Cc77BI,iCACE,iBd47BN,Cc77BI,uBAEE,gBd27BN,Ccx7BM,iCACE,ed07BR,Ccp7BE,kBACE,WAAA,CAIA,eAAA,CADA,mBAAA,CAFA,6BAAA,CACA,cAAA,CAGA,kBds7BJ,Ccl7BE,mBAEE,YAAA,CADA,adq7BJ,Cch7BE,sBACE,gBAAA,CACA,Udk7BJ,Cc76BA,gBACE,gDdg7BF,Cc76BE,uBACE,YAAA,CACA,cAAA,CACA,6BAAA,CACA,ad+6BJ,Cc36BE,kCACE,sCd66BJ,Cc16BI,gFACE,+Bd46BN,Ccp6BA,cAKE,wCAAA,CADA,gBAAA,CADA,iBAAA,CADA,eAAA,CADA,Ud26BF,CK/3BI,mCS7CJ,cASI,Udu6BF,CACF,Ccn6BE,yBACE,sCdq6BJ,Cc95BA,WACE,mBAAA,CACA,SAAA,CAEA,cAAA,CADA,qBdk6BF,CK94BI,mCSvBJ,WAQI,edi6BF,CACF,Cc95BE,iBACE,oBAAA,CAEA,aAAA,CACA,iBAAA,CAFA,Ydk6BJ,Cc75BI,wBACE,ed+5BN,Cc35BI,qBAGE,iBAAA,CAFA,gBAAA,CACA,mBd85BN,CepkCE,uBAME,kBAAA,CACA,mBAAA,CAHA,gCAAA,CACA,cAAA,CAJA,oBAAA,CAEA,eAAA,CADA,kBAAA,CAMA,gEfukCJ,CejkCI,gCAEE,2CAAA,CACA,uCAAA,CAFA,gCfqkCN,Ce/jCI,0DAEE,0CAAA,CACA,sCAAA,CAFA,+BfmkCN,Ce5jCE,gCAKE,4BfikCJ,CetkCE,gEAME,6BfgkCJ,CetkCE,gCAME,4BfgkCJ,CetkCE,sBAIE,6DAAA,CAGA,8BAAA,CAJA,eAAA,CAFA,aAAA,CACA,eAAA,CAMA,sCf8jCJ,CezjCI,wDACE,6CAAA,CACA,8Bf2jCN,CevjCI,+BACE,UfyjCN,CgB5mCA,WAOE,2CAAA,CAGA,8CACE,CALF,gCAAA,CADA,aAAA,CAHA,MAAA,CADA,eAAA,CACA,OAAA,CACA,KAAA,CACA,ShBmnCF,CgBxmCE,aAfF,WAgBI,YhB2mCF,CACF,CgBxmCE,mBAIE,2BAAA,CAHA,iEhB2mCJ,CgBpmCE,mBACE,kDACE,CAEF,kEhBomCJ,CgB9lCE,kBAEE,kBAAA,CADA,YAAA,CAEA,ehBgmCJ,CgB5lCE,mBAKE,kBAAA,CAEA,cAAA,CAHA,YAAA,CAIA,uCAAA,CALA,aAAA,CAFA,iBAAA,CAQA,uBAAA,CAHA,qBAAA,CAJA,ShBqmCJ,CgB3lCI,yBACE,UhB6lCN,CgBzlCI,iCACE,oBhB2lCN,CgBvlCI,uCAEE,uCAAA,CADA,YhB0lCN,CgBrlCI,2BAEE,YAAA,CADA,ahBwlCN,CK1+BI,0CW/GA,2BAMI,YhBulCN,CACF,CgBplCM,8DAIE,iBAAA,CAHA,aAAA,CAEA,aAAA,CADA,UhBwlCR,CKxgCI,mCWzEA,iCAII,YhBilCN,CACF,CgB9kCM,wCACE,YhBglCR,CgB5kCM,+CACE,oBhB8kCR,CKnhCI,sCWtDA,iCAII,YhBykCN,CACF,CgBpkCE,kBAEE,YAAA,CACA,cAAA,CAFA,iBAAA,CAIA,8DACE,CAFF,kBhBukCJ,CgBjkCI,oCAGE,SAAA,CADA,mBAAA,CAKA,6BAAA,CAHA,8DACE,CAJF,UhBukCN,CgB9jCM,8CACE,8BhBgkCR,CgB3jCI,8BACE,ehB6jCN,CgBxjCE,4BAGE,gBAAA,CAAA,kBhB4jCJ,CgB/jCE,4BAGE,iBAAA,CAAA,iBhB4jCJ,CgB/jCE,kBACE,WAAA,CAGA,eAAA,CAFA,aAAA,CAGA,kBhB0jCJ,CgBvjCI,4CAGE,SAAA,CADA,mBAAA,CAKA,8BAAA,CAHA,8DACE,CAJF,UhB6jCN,CgBpjCM,sDACE,6BhBsjCR,CgBljCM,8DAGE,SAAA,CADA,mBAAA,CAKA,uBAAA,CAHA,8DACE,CAJF,ShBwjCR,CgB7iCI,uCAGE,WAAA,CAFA,iBAAA,CACA,UhBgjCN,CgB1iCE,mBACE,YAAA,CACA,aAAA,CACA,cAAA,CAEA,+CACE,CAFF,kBhB6iCJ,CgBviCI,8DACE,WAAA,CACA,SAAA,CACA,oChByiCN,CgBhiCI,yBACE,QhBkiCN,CgB7hCE,mBACE,YhB+hCJ,CK3lCI,mCW2DF,6BAQI,gBhB+hCJ,CgBviCA,6BAQI,iBhB+hCJ,CgBviCA,mBAKI,aAAA,CAEA,iBAAA,CADA,ahBiiCJ,CACF,CKnmCI,sCW2DF,6BAaI,kBhB+hCJ,CgB5iCA,6BAaI,mBhB+hCJ,CACF,CD9wCA,SAGE,uCAAA,CAFA,eAAA,CACA,eCkxCF,CD9wCE,eACE,mBAAA,CACA,cAAA,CAGA,eAAA,CADA,QAAA,CADA,SCkxCJ,CD5wCE,sCAEE,WAAA,CADA,iBAAA,CAAA,kBC+wCJ,CD1wCE,eACE,+BC4wCJ,CDzwCI,0CACE,+BC2wCN,CDrwCA,UAKE,wBkBaa,ClBZb,oBAAA,CAFA,UAAA,CAHA,oBAAA,CAEA,eAAA,CADA,0BAAA,CAAA,2BC4wCF,CkB9yCA,MACE,0MAAA,CACA,gMAAA,CACA,yNlBizCF,CkB3yCA,QACE,eAAA,CACA,elB8yCF,CkB3yCE,eAKE,uCAAA,CAJA,aAAA,CAGA,eAAA,CADA,eAAA,CADA,eAAA,CAIA,sBlB6yCJ,CkB1yCI,+BACE,YlB4yCN,CkBzyCM,mCAEE,WAAA,CADA,UlB4yCR,CkBpyCQ,sFAME,iBAAA,CALA,aAAA,CAGA,aAAA,CADA,cAAA,CAEA,kBAAA,CAHA,UlB0yCV,CkB/xCE,cAGE,eAAA,CADA,QAAA,CADA,SlBmyCJ,CkB7xCE,cAGE,sBAAA,CAFA,YAAA,CACA,SAAA,CAEA,iBAAA,CAEA,uBAAA,CADA,sBlBgyCJ,CkB5xCI,sBACE,uClB8xCN,CkBvxCM,6EAEE,+BlByxCR,CkBpxCI,2BAIE,iBlBmxCN,CkB/wCI,4CACE,gBlBixCN,CkBlxCI,4CACE,iBlBixCN,CkB7wCI,kBAGE,iBAAA,CAFA,aAAA,CACA,YlBgxCN,CkB3wCI,sGACE,+BAAA,CACA,clB6wCN,CkBzwCI,4BACE,uCAAA,CACA,oBlB2wCN,CkBvwCI,0CACE,YlBywCN,CkBtwCM,yDAKE,6BAAA,CAJA,aAAA,CAEA,WAAA,CACA,qCAAA,CAAA,6BAAA,CAFA,UlB2wCR,CkBpwCM,kDACE,YlBswCR,CkBhwCE,iCACE,YlBkwCJ,CkB/vCI,6CACE,WAAA,CAGA,WlB+vCN,CkB1vCE,cACE,alB4vCJ,CkBxvCE,gBACE,YlB0vCJ,CKxtCI,0Ca3BA,0CASE,2CAAA,CAHA,YAAA,CACA,qBAAA,CACA,WAAA,CALA,MAAA,CADA,iBAAA,CACA,OAAA,CACA,KAAA,CACA,SlByvCJ,CkB9uCI,+DACE,eAAA,CACA,elBgvCN,CkB5uCI,gCAQE,qDAAA,CAHA,uCAAA,CAEA,cAAA,CALA,aAAA,CAEA,kBAAA,CADA,wBAAA,CAFA,iBAAA,CAKA,kBlBgvCN,CkB3uCM,wDAGE,UlBivCR,CkBpvCM,wDAGE,WlBivCR,CkBpvCM,8CAIE,aAAA,CAEA,aAAA,CACA,YAAA,CANA,iBAAA,CACA,SAAA,CAGA,YlB+uCR,CkB1uCQ,oDAKE,6BAAA,CADA,UAAA,CAHA,aAAA,CAEA,WAAA,CAGA,2CAAA,CAAA,mCAAA,CACA,4BAAA,CAAA,oBAAA,CACA,6BAAA,CAAA,qBAAA,CACA,yBAAA,CAAA,iBAAA,CAPA,UlBmvCV,CkBvuCM,8CAGE,2CAAA,CACA,gEACE,CAJF,eAAA,CAKA,4BAAA,CAJA,kBlB4uCR,CkBruCQ,2DACE,YlBuuCV,CkBluCM,8CAGE,2CAAA,CADA,gCAAA,CADA,elBsuCR,CkBhuCM,yCAIE,aAAA,CAFA,UAAA,CAIA,YAAA,CADA,aAAA,CAJA,iBAAA,CACA,WAAA,CACA,SlBquCR,CkB7tCI,+BACE,MlB+tCN,CkB3tCI,+BACE,4DlB6tCN,CkB1tCM,qDACE,+BlB4tCR,CkBztCQ,sHACE,+BlB2tCV,CkBrtCI,+BAEE,YAAA,CADA,mBlBwtCN,CkBptCM,mCACE,elBstCR,CkBltCM,6CACE,SlBotCR,CkBhtCM,uDAGE,mBlBmtCR,CkBttCM,uDAGE,kBlBmtCR,CkBttCM,6CAIE,gBAAA,CAFA,aAAA,CADA,YlBqtCR,CkB/sCQ,mDAKE,6BAAA,CADA,UAAA,CAHA,aAAA,CAEA,WAAA,CAGA,2CAAA,CAAA,mCAAA,CACA,4BAAA,CAAA,oBAAA,CACA,6BAAA,CAAA,qBAAA,CACA,yBAAA,CAAA,iBAAA,CAPA,UlBwtCV,CkBxsCM,+CACE,mBlB0sCR,CkBlsCM,4CAEE,wBAAA,CADA,elBqsCR,CkBjsCQ,oEACE,mBlBmsCV,CkBpsCQ,oEACE,oBlBmsCV,CkB/rCQ,4EACE,iBlBisCV,CkBlsCQ,4EACE,kBlBisCV,CkB7rCQ,oFACE,mBlB+rCV,CkBhsCQ,oFACE,oBlB+rCV,CkB3rCQ,4FACE,mBlB6rCV,CkB9rCQ,4FACE,oBlB6rCV,CkBtrCE,mBACE,wBlBwrCJ,CkBprCE,wBACE,YAAA,CACA,SAAA,CAIA,0BAAA,CAHA,oElBurCJ,CkBjrCI,kCACE,2BlBmrCN,CkB9qCE,gCACE,SAAA,CAIA,uBAAA,CAHA,qElBirCJ,CkB3qCI,8CAEE,kCAAA,CAAA,0BlB4qCN,CACF,CK32CI,0CauMA,0CACE,YlBuqCJ,CkBpqCI,yDACE,UlBsqCN,CkBlqCI,wDACE,YlBoqCN,CkBhqCI,kDACE,YlBkqCN,CkB7pCE,gBAIE,iDAAA,CADA,gCAAA,CAFA,aAAA,CACA,elBiqCJ,CACF,CKx6CM,+DagRF,6CACE,YlB2pCJ,CkBxpCI,4DACE,UlB0pCN,CkBtpCI,2DACE,YlBwpCN,CkBppCI,qDACE,YlBspCN,CACF,CKh6CI,mCa7JJ,QA6aI,oBlBopCF,CkB9oCI,kCAME,qCAAA,CACA,qDAAA,CANA,eAAA,CACA,KAAA,CAGA,SlBgpCN,CkB3oCM,6CACE,uBlB6oCR,CkBzoCM,gDACE,YlB2oCR,CkBtoCI,2CACE,kBlByoCN,CkB1oCI,2CACE,mBlByoCN,CkB1oCI,iCAEE,oBlBwoCN,CkBjoCI,yDACE,kBlBmoCN,CkBpoCI,yDACE,iBlBmoCN,CACF,CKz7CI,sCa7JJ,QAydI,oBAAA,CACA,oDlBioCF,CkB3nCI,gCAME,qCAAA,CACA,qDAAA,CANA,eAAA,CACA,KAAA,CAGA,SlB6nCN,CkBxnCM,8CACE,uBlB0nCR,CkBtnCM,8CACE,YlBwnCR,CkBnnCI,yCACE,kBlBsnCN,CkBvnCI,yCACE,mBlBsnCN,CkBvnCI,+BAEE,oBlBqnCN,CkB9mCI,uDACE,kBlBgnCN,CkBjnCI,uDACE,iBlBgnCN,CkB3mCE,wBACE,YAAA,CACA,sBAAA,CAEA,SAAA,CACA,6FACE,CAHF,mBlB+mCJ,CkBvmCI,sCACE,elBymCN,CkBpmCE,iFACE,sBAAA,CAEA,SAAA,CACA,4FACE,CAHF,kBlBwmCJ,CkB/lCE,iDACE,elBimCJ,CkB7lCE,6CACE,YlB+lCJ,CkB3lCE,uBACE,aAAA,CACA,elB6lCJ,CkB1lCI,kCACE,elB4lCN,CkBxlCI,qCACE,elB0lCN,CkBvlCM,0CACE,uClBylCR,CkBrlCM,6DACE,mBlBulCR,CkBnlCM,yFAEE,YlBqlCR,CkBhlCI,yCAEE,kBlBolCN,CkBtlCI,yCAEE,mBlBolCN,CkBtlCI,+BACE,aAAA,CAGA,SAAA,CADA,kBlBmlCN,CkB/kCM,2DACE,SlBilCR,CkB3kCE,cAGE,kBAAA,CADA,YAAA,CAEA,gCAAA,CAHA,WlBglCJ,CkB1kCI,oBACE,uDlB4kCN,CkBxkCI,oBAME,6BAAA,CACA,kBAAA,CAFA,UAAA,CAJA,oBAAA,CAEA,WAAA,CAMA,2CAAA,CAAA,mCAAA,CACA,4BAAA,CAAA,oBAAA,CACA,6BAAA,CAAA,qBAAA,CACA,yBAAA,CAAA,iBAAA,CAJA,yBAAA,CAJA,qBAAA,CAFA,UlBolCN,CkBvkCM,8BACE,wBlBykCR,CkBrkCM,kKAEE,uBlBskCR,CkBxjCI,2EACE,YlB6jCN,CkB1jCM,oDACE,alB4jCR,CkBzjCQ,kEAKE,qCAAA,CACA,qDAAA,CAFA,YAAA,CAHA,eAAA,CACA,KAAA,CACA,SlB8jCV,CkBxjCU,0FACE,mBlB0jCZ,CkBrjCQ,0EACE,QlBujCV,CkBljCM,sFACE,kBlBojCR,CkBrjCM,sFACE,mBlBojCR,CkBhjCM,kDACE,uClBkjCR,CkB5iCI,2CACE,sBAAA,CAEA,SAAA,CADA,kBlB+iCN,CkBtiCI,qFAIE,mDlByiCN,CkB7iCI,qFAIE,oDlByiCN,CkB7iCI,2EACE,aAAA,CACA,oBAAA,CAGA,SAAA,CAFA,kBlB0iCN,CkBriCM,yFAEE,gBAAA,CADA,gBlBwiCR,CkBniCM,0FACE,YlBqiCR,CACF,CmBzvDA,eAKE,eAAA,CACA,eAAA,CAJA,SnBgwDF,CmBzvDE,gCANA,kBAAA,CAFA,YAAA,CAGA,sBnBuwDF,CmBlwDE,iBAOE,mBAAA,CAFA,aAAA,CADA,gBAAA,CAEA,iBnB4vDJ,CmBvvDE,wBAEE,qDAAA,CADA,uCnB0vDJ,CmBrvDE,qBACE,6CnBuvDJ,CmBlvDI,sDAEE,uDAAA,CADA,+BnBqvDN,CmBjvDM,8DACE,+BnBmvDR,CmB9uDI,mCACE,uCAAA,CACA,oBnBgvDN,CmB5uDI,yBAKE,iBAAA,CADA,yCAAA,CAHA,aAAA,CAEA,eAAA,CADA,YnBivDN,CoBjyDE,eAGE,+DAAA,CADA,oBAAA,CADA,qBpBsyDJ,CKjnDI,0CetLF,eAOI,YpBoyDJ,CACF,CoB9xDM,6BACE,oBpBgyDR,CoB1xDE,kBACE,YAAA,CACA,qBAAA,CACA,SAAA,CACA,qBpB4xDJ,CoBrxDI,0BACE,sBpBuxDN,CoBpxDM,gEACE,+BpBsxDR,CoBhxDE,gBAEE,uCAAA,CADA,epBmxDJ,CoB9wDE,kBACE,oBpBgxDJ,CoB7wDI,mCAGE,kBAAA,CAFA,YAAA,CACA,SAAA,CAEA,iBpB+wDN,CoB3wDI,oCAIE,kBAAA,CAHA,mBAAA,CACA,kBAAA,CACA,SAAA,CAGA,QAAA,CADA,iBpB8wDN,CoBzwDI,0DACE,kBpB2wDN,CoB5wDI,0DACE,iBpB2wDN,CoBvwDI,iDACE,uBAAA,CAEA,YpBwwDN,CoBnwDE,4BACE,YpBqwDJ,CoB9vDA,YAGE,kBAAA,CAFA,YAAA,CAIA,eAAA,CAHA,SAAA,CAIA,eAAA,CAFA,UpBmwDF,CoB9vDE,yBACE,WpBgwDJ,CoBzvDA,kBACE,YpB4vDF,CKprDI,0CezEJ,kBAKI,wBpB4vDF,CACF,CoBzvDE,qCACE,WpB2vDJ,CK/sDI,sCe7CF,+CAKI,kBpB2vDJ,CoBhwDA,+CAKI,mBpB2vDJ,CACF,CKjsDI,0CerDJ,6BAMI,SAAA,CAFA,eAAA,CACA,UpBwvDF,CoBrvDE,qDACE,gBpBuvDJ,CoBpvDE,gDACE,SpBsvDJ,CoBnvDE,4CACE,iBAAA,CAAA,kBpBqvDJ,CoBlvDE,2CAEE,WAAA,CADA,cpBqvDJ,CoBjvDE,2CACE,mBAAA,CACA,cAAA,CACA,SAAA,CACA,oBAAA,CAAA,iBpBmvDJ,CoBhvDE,2CACE,SpBkvDJ,CoB/uDE,qCAEE,WAAA,CACA,eAAA,CAFA,epBmvDJ,CACF,CqB75DA,MACE,qBAAA,CACA,yBrBg6DF,CqB15DA,aAME,qCAAA,CADA,cAAA,CAEA,0FACE,CAPF,cAAA,CACA,KAAA,CAaA,mDAAA,CACA,qBAAA,CAJA,wFACE,CATF,UAAA,CADA,SrBo6DF,CsB/6DA,MACE,igBtBk7DF,CsB56DA,WACE,iBtB+6DF,CKjxDI,mCiB/JJ,WAKI,etB+6DF,CACF,CsB56DE,kBACE,YtB86DJ,CsB16DE,oBAEE,SAAA,CADA,StB66DJ,CK1wDI,0CiBpKF,8BAkBI,YtB06DJ,CsB57DA,8BAkBI,atB06DJ,CsB57DA,oBAYI,2CAAA,CACA,kBAAA,CAJA,WAAA,CACA,eAAA,CACA,mBAAA,CALA,iBAAA,CACA,SAAA,CAUA,uBAAA,CAHA,4CACE,CAPF,UtBo7DJ,CsBv6DI,+DACE,SAAA,CACA,oCtBy6DN,CACF,CKhzDI,mCiBjJF,8BAyCI,MtBm6DJ,CsB58DA,8BAyCI,OtBm6DJ,CsB58DA,oBAoCI,0BAAA,CADA,cAAA,CADA,QAAA,CAHA,cAAA,CACA,KAAA,CAKA,sDACE,CALF,OtB26DJ,CsBh6DI,+DAME,YAAA,CACA,SAAA,CACA,4CACE,CARF,UtBq6DN,CACF,CK/yDI,0CiBxGA,+DAII,mBtBu5DN,CACF,CK71DM,+DiB/DF,+DASI,mBtBu5DN,CACF,CKl2DM,+DiB/DF,+DAcI,mBtBu5DN,CACF,CsBl5DE,kBAEE,kCAAA,CAAA,0BtBm5DJ,CKj0DI,0CiBpFF,4BAmBI,MtB+4DJ,CsBl6DA,4BAmBI,OtB+4DJ,CsBl6DA,kBAUI,QAAA,CAEA,SAAA,CADA,eAAA,CALA,cAAA,CACA,KAAA,CAWA,wBAAA,CALA,qGACE,CALF,OAAA,CADA,StB05DJ,CsB54DI,4BACE,yBtB84DN,CsB14DI,6DAEE,WAAA,CACA,SAAA,CAMA,uBAAA,CALA,sGACE,CAJF,UtBg5DN,CACF,CK52DI,mCiBjEF,4BA2CI,WtB04DJ,CsBr7DA,4BA2CI,UtB04DJ,CsBr7DA,kBA6CI,eAAA,CAHA,iBAAA,CAIA,8CAAA,CAFA,atBy4DJ,CACF,CK34DM,+DiBOF,6DAII,atBo4DN,CACF,CK13DI,sCiBfA,6DASI,atBo4DN,CACF,CsB/3DE,iBAIE,2CAAA,CACA,0BAAA,CAFA,aAAA,CAFA,iBAAA,CAKA,2CACE,CALF,StBq4DJ,CKv4DI,mCiBAF,iBAaI,0BAAA,CACA,mBAAA,CAFA,atBi4DJ,CsB53DI,uBACE,0BtB83DN,CACF,CsB13DI,4DAEE,2CAAA,CACA,6BAAA,CACA,8BAAA,CAHA,gCtB+3DN,CsBv3DE,4BAKE,mBAAA,CAAA,oBtB43DJ,CsBj4DE,4BAKE,mBAAA,CAAA,oBtB43DJ,CsBj4DE,kBAQE,gBAAA,CAFA,eAAA,CAFA,WAAA,CAHA,iBAAA,CAMA,sBAAA,CAJA,UAAA,CADA,StB+3DJ,CsBt3DI,+BACE,qBtBw3DN,CsBp3DI,kEAEE,uCtBq3DN,CsBj3DI,6BACE,YtBm3DN,CKv5DI,0CiBaF,kBA8BI,eAAA,CADA,aAAA,CADA,UtBo3DJ,CACF,CKj7DI,mCiBgCF,4BAmCI,mBtBo3DJ,CsBv5DA,4BAmCI,oBtBo3DJ,CsBv5DA,kBAqCI,aAAA,CADA,etBm3DJ,CsB/2DI,+BACE,uCtBi3DN,CsB72DI,mCACE,gCtB+2DN,CsB32DI,6DACE,kBtB62DN,CsB12DM,8EACE,uCtB42DR,CsBx2DM,0EACE,WtB02DR,CACF,CsBp2DE,iBAIE,cAAA,CAHA,oBAAA,CAEA,aAAA,CAEA,kCACE,CAJF,YtBy2DJ,CsBj2DI,uBACE,UtBm2DN,CsB/1DI,yCAGE,UtBk2DN,CsBr2DI,yCAGE,WtBk2DN,CsBr2DI,+BACE,iBAAA,CACA,SAAA,CAEA,StBi2DN,CsB91DM,6CACE,oBtBg2DR,CKv8DI,0CiB+FA,yCAcI,UtB+1DN,CsB72DE,yCAcI,WtB+1DN,CsB72DE,+BAaI,StBg2DN,CsB51DM,+CACE,YtB81DR,CACF,CKn+DI,mCiBkHA,+BAwBI,mBtB61DN,CsB11DM,8CACE,YtB41DR,CACF,CsBt1DE,8BAGE,WtB01DJ,CsB71DE,8BAGE,UtB01DJ,CsB71DE,oBAKE,mBAAA,CAJA,iBAAA,CACA,SAAA,CAEA,StBy1DJ,CK/9DI,0CiBkIF,8BAUI,WtBw1DJ,CsBl2DA,8BAUI,UtBw1DJ,CsBl2DA,oBASI,StBy1DJ,CACF,CsBr1DI,uCACE,iBtB21DN,CsB51DI,uCACE,kBtB21DN,CsB51DI,6BAEE,uCAAA,CACA,SAAA,CAIA,oBAAA,CAHA,+DtBw1DN,CsBl1DM,iDAEE,uCAAA,CADA,YtBq1DR,CsBh1DM,gGAGE,SAAA,CADA,mBAAA,CAEA,kBtBi1DR,CsB90DQ,sGACE,UtBg1DV,CsBz0DE,8BAOE,mBAAA,CAAA,oBtBg1DJ,CsBv1DE,8BAOE,mBAAA,CAAA,oBtBg1DJ,CsBv1DE,oBAIE,kBAAA,CAKA,yCAAA,CANA,YAAA,CAKA,eAAA,CAFA,WAAA,CAKA,SAAA,CAVA,iBAAA,CACA,KAAA,CAUA,uBAAA,CAFA,kBAAA,CALA,UtBk1DJ,CKzhEI,mCiBkMF,8BAgBI,mBtB40DJ,CsB51DA,8BAgBI,oBtB40DJ,CsB51DA,oBAiBI,etB20DJ,CACF,CsBx0DI,+DACE,SAAA,CACA,0BtB00DN,CsBr0DE,6BAKE,+BtBw0DJ,CsB70DE,0DAME,gCtBu0DJ,CsB70DE,6BAME,+BtBu0DJ,CsB70DE,mBAIE,eAAA,CAHA,iBAAA,CAEA,UAAA,CADA,StB20DJ,CKxhEI,0CiB2MF,mBAWI,QAAA,CADA,UtBw0DJ,CACF,CKjjEI,mCiB8NF,mBAiBI,SAAA,CADA,UAAA,CAEA,sBtBu0DJ,CsBp0DI,8DACE,8BAAA,CACA,StBs0DN,CACF,CsBj0DE,uBASE,kCAAA,CAAA,0BAAA,CAFA,2CAAA,CANA,WAAA,CACA,eAAA,CAIA,kBtBk0DJ,CsB5zDI,iEAZF,uBAaI,uBtB+zDJ,CACF,CK9lEM,+DiBiRJ,uBAkBI,atB+zDJ,CACF,CK7kEI,sCiB2PF,uBAuBI,atB+zDJ,CACF,CKllEI,mCiB2PF,uBA4BI,YAAA,CAEA,yDAAA,CADA,oBtBg0DJ,CsB5zDI,kEACE,etB8zDN,CsB1zDI,6BACE,+CtB4zDN,CsBxzDI,0CAEE,YAAA,CADA,WtB2zDN,CsBtzDI,gDACE,oDtBwzDN,CsBrzDM,sDACE,0CtBuzDR,CACF,CsBhzDA,kBACE,gCAAA,CACA,qBtBmzDF,CsBhzDE,wBAKE,qDAAA,CADA,uCAAA,CAFA,gBAAA,CACA,kBAAA,CAFA,eAAA,CAKA,uBtBkzDJ,CKtnEI,mCiB8TF,kCAUI,mBtBkzDJ,CsB5zDA,kCAUI,oBtBkzDJ,CACF,CsB9yDE,wBAGE,eAAA,CADA,QAAA,CADA,SAAA,CAIA,wBAAA,CAAA,gBtB+yDJ,CsB3yDE,wBACE,yDtB6yDJ,CsB1yDI,oCACE,etB4yDN,CsBvyDE,wBACE,aAAA,CACA,YAAA,CAEA,uBAAA,CADA,gCtB0yDJ,CsBtyDI,4DACE,uDtBwyDN,CsBpyDI,gDACE,mBtBsyDN,CsBjyDE,gCAKE,cAAA,CADA,aAAA,CAEA,YAAA,CALA,eAAA,CAMA,uBAAA,CALA,KAAA,CACA,StBuyDJ,CsBhyDI,wCACE,YtBkyDN,CsB7xDI,wDACE,YtB+xDN,CsB3xDI,oCAGE,+BAAA,CADA,gBAAA,CADA,mBAAA,CAGA,2CtB6xDN,CKxqEI,mCiBuYA,8CAUI,mBtB2xDN,CsBryDE,8CAUI,oBtB2xDN,CACF,CsBvxDI,oFAEE,uDAAA,CADA,+BtB0xDN,CsBpxDE,sCACE,2CtBsxDJ,CsBjxDE,2BAGE,eAAA,CADA,eAAA,CADA,iBtBqxDJ,CKzrEI,mCiBmaF,qCAOI,mBtBmxDJ,CsB1xDA,qCAOI,oBtBmxDJ,CACF,CsB/wDE,kCAEE,MtBqxDJ,CsBvxDE,kCAEE,OtBqxDJ,CsBvxDE,wBAME,uCAAA,CAFA,aAAA,CACA,YAAA,CAJA,iBAAA,CAEA,YtBoxDJ,CKnrEI,0CiB4ZF,wBAUI,YtBixDJ,CACF,CsB9wDI,8BAKE,6BAAA,CADA,UAAA,CAHA,oBAAA,CAEA,WAAA,CAGA,+CAAA,CAAA,uCAAA,CACA,4BAAA,CAAA,oBAAA,CACA,6BAAA,CAAA,qBAAA,CACA,yBAAA,CAAA,iBAAA,CAPA,UtBuxDN,CsB7wDM,wCACE,oBtB+wDR,CsBzwDE,8BAGE,uCAAA,CAFA,gBAAA,CACA,etB4wDJ,CsBxwDI,iCAKE,gCAAA,CAHA,eAAA,CACA,eAAA,CACA,eAAA,CAHA,etB8wDN,CsBvwDM,sCACE,oBtBywDR,CsBpwDI,iCAKE,gCAAA,CAHA,gBAAA,CACA,eAAA,CACA,eAAA,CAHA,atB0wDN,CsBnwDM,sCACE,oBtBqwDR,CsB/vDE,yBAKE,gCAAA,CAJA,aAAA,CAEA,gBAAA,CACA,iBAAA,CAFA,atBowDJ,CsB7vDE,uBAGE,wBAAA,CAFA,+BAAA,CACA,yBtBgwDJ,CuBp6EA,WACE,iBAAA,CACA,SvBu6EF,CuBp6EE,kBAOE,2CAAA,CACA,mBAAA,CACA,8BAAA,CAHA,gCAAA,CAHA,QAAA,CAEA,gBAAA,CADA,YAAA,CAMA,SAAA,CATA,iBAAA,CACA,sBAAA,CAaA,mCAAA,CAJA,oEvBu6EJ,CuBh6EI,6EACE,gBAAA,CACA,SAAA,CAKA,+BAAA,CAJA,8EvBm6EN,CuB35EI,wBAWE,+BAAA,CAAA,8CAAA,CAFA,6BAAA,CAAA,8BAAA,CACA,YAAA,CAFA,UAAA,CAHA,QAAA,CAFA,QAAA,CAIA,kBAAA,CADA,iBAAA,CALA,iBAAA,CACA,KAAA,CAEA,OvBo6EN,CuBx5EE,iBAOE,mBAAA,CAFA,eAAA,CACA,oBAAA,CAHA,QAAA,CAFA,kBAAA,CAGA,aAAA,CAFA,SvB+5EJ,CuBt5EE,iBACE,kBvBw5EJ,CuBp5EE,2BAGE,kBAAA,CAAA,oBvB05EJ,CuB75EE,2BAGE,mBAAA,CAAA,mBvB05EJ,CuB75EE,iBAIE,cAAA,CAHA,aAAA,CAIA,YAAA,CAIA,uBAAA,CAHA,2CACE,CALF,UvB25EJ,CuBj5EI,8CACE,+BvBm5EN,CuB/4EI,uBACE,qDvBi5EN,CwBr+EA,YAIE,qBAAA,CADA,aAAA,CAGA,gBAAA,CALA,eAAA,CACA,UAAA,CAGA,axBy+EF,CwBr+EE,aATF,YAUI,YxBw+EF,CACF,CK1zEI,0CmB3KF,+BAeI,axBm+EJ,CwBl/EA,+BAeI,cxBm+EJ,CwBl/EA,qBAUI,2CAAA,CAHA,aAAA,CAEA,WAAA,CALA,cAAA,CACA,KAAA,CASA,uBAAA,CAHA,iEACE,CAJF,aAAA,CAFA,SxB4+EJ,CwBh+EI,mEACE,8BAAA,CACA,6BxBk+EN,CwB/9EM,6EACE,8BxBi+ER,CwB59EI,6CAEE,QAAA,CAAA,MAAA,CACA,QAAA,CAEA,eAAA,CAJA,iBAAA,CACA,OAAA,CAEA,qBAAA,CAFA,KxBi+EN,CACF,CKz2EI,sCmBtKJ,YAuDI,QxB49EF,CwBz9EE,mBACE,WxB29EJ,CwBv9EE,6CACE,UxBy9EJ,CACF,CwBr9EE,uBACE,YAAA,CACA,OxBu9EJ,CKx3EI,mCmBjGF,uBAMI,QxBu9EJ,CwBp9EI,8BACE,WxBs9EN,CwBl9EI,qCACE,axBo9EN,CwBh9EI,+CACE,kBxBk9EN,CACF,CwB78EE,wBAUE,uBAAA,CANA,kCAAA,CAAA,0BAAA,CAHA,cAAA,CACA,eAAA,CASA,yDAAA,CAFA,oBxB48EJ,CwBv8EI,2CAEE,YAAA,CADA,WxB08EN,CwBr8EI,mEACE,+CxBu8EN,CwBp8EM,qHACE,oDxBs8ER,CwBn8EQ,iIACE,0CxBq8EV,CwBt7EE,wCAGE,wBACE,qBxBs7EJ,CwBl7EE,6BACE,kCxBo7EJ,CwBr7EE,6BACE,iCxBo7EJ,CACF,CKh5EI,0CmB5BF,YAME,0BAAA,CADA,QAAA,CAEA,SAAA,CANA,cAAA,CACA,KAAA,CAMA,sDACE,CALF,OAAA,CADA,SxBq7EF,CwB16EE,4CAEE,WAAA,CACA,SAAA,CACA,4CACE,CAJF,UxB+6EJ,CACF,CyB5lFA,iBACE,GACE,QzB8lFF,CyB3lFA,GACE,azB6lFF,CACF,CyBzlFA,gBACE,GACE,SAAA,CACA,0BzB2lFF,CyBxlFA,IACE,SzB0lFF,CyBvlFA,GACE,SAAA,CACA,uBzBylFF,CACF,CyBjlFA,MACE,+eAAA,CACA,ygBAAA,CACA,mmBAAA,CACA,sfzBmlFF,CyB7kFA,WAOE,kCAAA,CAAA,0BAAA,CANA,aAAA,CACA,gBAAA,CACA,eAAA,CAEA,uCAAA,CAGA,uBAAA,CAJA,kBzBmlFF,CyB5kFE,iBACE,UzB8kFJ,CyB1kFE,iBACE,oBAAA,CAEA,aAAA,CACA,qBAAA,CAFA,UzB8kFJ,CyBzkFI,+BACE,iBzB4kFN,CyB7kFI,+BACE,kBzB4kFN,CyB7kFI,qBAEE,gBzB2kFN,CyBvkFI,kDACE,iBzB0kFN,CyB3kFI,kDACE,kBzB0kFN,CyB3kFI,kDAEE,iBzBykFN,CyB3kFI,kDAEE,kBzBykFN,CyBpkFE,iCAGE,iBzBykFJ,CyB5kFE,iCAGE,kBzBykFJ,CyB5kFE,uBACE,oBAAA,CACA,6BAAA,CAEA,eAAA,CACA,sBAAA,CACA,qBzBskFJ,CyBlkFE,kBACE,YAAA,CAMA,gBAAA,CALA,SAAA,CAMA,oBAAA,CAHA,gBAAA,CAIA,WAAA,CAHA,eAAA,CAFA,SAAA,CADA,UzB0kFJ,CyBjkFI,iDACE,4BzBmkFN,CyB9jFE,iBACE,eAAA,CACA,sBzBgkFJ,CyB7jFI,gDACE,2BzB+jFN,CyB3jFI,kCAIE,kBzBmkFN,CyBvkFI,kCAIE,iBzBmkFN,CyBvkFI,wBAOE,6BAAA,CADA,UAAA,CALA,oBAAA,CAEA,YAAA,CAKA,4BAAA,CAAA,oBAAA,CACA,6BAAA,CAAA,qBAAA,CACA,yBAAA,CAAA,iBAAA,CALA,uBAAA,CAHA,WzBqkFN,CyBzjFI,iCACE,azB2jFN,CyBvjFI,iCACE,gDAAA,CAAA,wCzByjFN,CyBrjFI,+BACE,8CAAA,CAAA,sCzBujFN,CyBnjFI,+BACE,8CAAA,CAAA,sCzBqjFN,CyBjjFI,sCACE,qDAAA,CAAA,6CzBmjFN,CyB7iFA,gBACE,YzBgjFF,CyB7iFE,gCAIE,kBzBijFJ,CyBrjFE,gCAIE,iBzBijFJ,CyBrjFE,sBAGE,kBAAA,CAGA,uCAAA,CALA,mBAAA,CAIA,gBAAA,CAHA,SzBmjFJ,CyB5iFI,+BACE,aAAA,CACA,oBzB8iFN,CyB1iFI,2CACE,UzB6iFN,CyB9iFI,2CACE,WzB6iFN,CyB9iFI,iCAEE,kBzB4iFN,CyBxiFI,0BACE,WzB0iFN,C0BjuFA,MACE,mSAAA,CACA,oVAAA,CACA,mOAAA,CACA,qZ1BouFF,C0B3tFE,iBAME,kDAAA,CADA,UAAA,CAJA,oBAAA,CAEA,cAAA,CAIA,mCAAA,CAAA,2BAAA,CACA,4BAAA,CAAA,oBAAA,CACA,6BAAA,CAAA,qBAAA,CACA,yBAAA,CAAA,iBAAA,CANA,0BAAA,CAFA,a1BsuFJ,C0B1tFE,uBACE,6B1B4tFJ,C0BxtFE,sBACE,wCAAA,CAAA,gC1B0tFJ,C0BttFE,6BACE,+CAAA,CAAA,uC1BwtFJ,C0BptFE,4BACE,8CAAA,CAAA,sC1BstFJ,C2BjwFA,SASE,2CAAA,CADA,gCAAA,CAJA,aAAA,CAGA,eAAA,CADA,aAAA,CADA,UAAA,CAFA,S3BwwFF,C2B/vFE,aAZF,SAaI,Y3BkwFF,CACF,CKvlFI,0CsBzLJ,SAkBI,Y3BkwFF,CACF,C2B/vFE,iBACE,mB3BiwFJ,C2B7vFE,yBAIE,iB3BowFJ,C2BxwFE,yBAIE,kB3BowFJ,C2BxwFE,eAQE,eAAA,CAPA,YAAA,CAMA,eAAA,CAJA,QAAA,CAEA,aAAA,CAHA,SAAA,CAWA,oBAAA,CAPA,kB3BkwFJ,C2BxvFI,kCACE,Y3B0vFN,C2BrvFE,eACE,aAAA,CACA,kBAAA,CAAA,mB3BuvFJ,C2BpvFI,sCACE,aAAA,CACA,S3BsvFN,C2BhvFE,eAOE,kCAAA,CAAA,0BAAA,CANA,YAAA,CAEA,eAAA,CADA,gBAAA,CAMA,UAAA,CAJA,uCAAA,CACA,oBAAA,CAIA,8D3BivFJ,C2B5uFI,0CACE,aAAA,CACA,S3B8uFN,C2B1uFI,6BAEE,kB3B6uFN,C2B/uFI,6BAEE,iB3B6uFN,C2B/uFI,mBAGE,iBAAA,CAFA,Y3B8uFN,C2BvuFM,2CACE,qB3ByuFR,C2B1uFM,2CACE,qB3B4uFR,C2B7uFM,2CACE,qB3B+uFR,C2BhvFM,2CACE,qB3BkvFR,C2BnvFM,2CACE,oB3BqvFR,C2BtvFM,2CACE,qB3BwvFR,C2BzvFM,2CACE,qB3B2vFR,C2B5vFM,2CACE,qB3B8vFR,C2B/vFM,4CACE,qB3BiwFR,C2BlwFM,4CACE,oB3BowFR,C2BrwFM,4CACE,qB3BuwFR,C2BxwFM,4CACE,qB3B0wFR,C2B3wFM,4CACE,qB3B6wFR,C2B9wFM,4CACE,qB3BgxFR,C2BjxFM,4CACE,oB3BmxFR,C2B7wFI,gCACE,SAAA,CAIA,yBAAA,CAHA,wC3BgxFN,C4Bn3FA,MACE,wS5Bs3FF,C4B72FE,mCACE,mBAAA,CACA,cAAA,CACA,QAAA,CAEA,mBAAA,CADA,kB5Bi3FJ,C4B52FE,oBAGE,kBAAA,CAOA,+CAAA,CACA,oBAAA,CAVA,mBAAA,CAIA,gBAAA,CACA,0BAAA,CACA,eAAA,CALA,QAAA,CAOA,qBAAA,CADA,eAAA,CAJA,wB5Bq3FJ,C4B32FI,0BAGE,uCAAA,CAFA,aAAA,CACA,YAAA,CAEA,6C5B62FN,C4Bx2FM,gEAEE,0CAAA,CADA,+B5B22FR,C4Br2FI,yBACE,uB5Bu2FN,C4B/1FI,gCAME,oDAAA,CADA,UAAA,CAJA,oBAAA,CAEA,YAAA,CAKA,qCAAA,CAAA,6BAAA,CACA,4BAAA,CAAA,oBAAA,CACA,6BAAA,CAAA,qBAAA,CACA,yBAAA,CAAA,iBAAA,CAJA,iCAAA,CAHA,0BAAA,CAFA,W5B02FN,C4B71FI,wFACE,0C5B+1FN,C6Bz6FA,iBACE,GACE,oB7B46FF,C6Bz6FA,IACE,kB7B26FF,C6Bx6FA,GACE,oB7B06FF,CACF,C6Bl6FA,MACE,0NAAA,CACA,uPAAA,CACA,wB7Bo6FF,C6B95FA,YA6BE,kCAAA,CAAA,0BAAA,CAVA,2CAAA,CACA,mBAAA,CACA,8BAAA,CAHA,gCAAA,CADA,sCAAA,CAdA,+IACE,CAYF,8BAAA,CAMA,SAAA,CArBA,iBAAA,CACA,uBAAA,CAyBA,4BAAA,CAJA,uDACE,CATF,6BAAA,CADA,S7Bk6FF,C6Bh5FE,oBAEE,SAAA,CAKA,uBAAA,CAJA,2EACE,CAHF,S7Bq5FJ,C6B34FE,oBAEE,eAAA,CACA,wBAAA,CAAA,gBAAA,CAFA,U7B+4FJ,C6B14FI,6CACE,qC7B44FN,C6Bx4FI,uCAEE,eAAA,CADA,mB7B24FN,C6Br4FI,6BACE,Y7Bu4FN,C6Bl4FE,8CACE,sC7Bo4FJ,C6Bh4FE,mBAEE,gBAAA,CADA,a7Bm4FJ,C6B/3FI,2CACE,Y7Bi4FN,C6B73FI,0CACE,e7B+3FN,C6Bv3FA,eACE,eAAA,CAGA,YAAA,CADA,0BAAA,CADA,kB7B43FF,C6Bv3FE,yBACE,a7By3FJ,C6Br3FE,oBACE,sCAAA,CACA,iB7Bu3FJ,C6Bn3FE,6BACE,oBAAA,CAGA,gB7Bm3FJ,C6B/2FE,sBAmBE,mBAAA,CAbA,cAAA,CAHA,oBAAA,CACA,gBAAA,CAAA,iBAAA,CAIA,YAAA,CAUA,eAAA,CAjBA,iBAAA,CAMA,wBAAA,CAAA,gBAAA,CAFA,uBAAA,CAHA,S7By3FJ,C6B/2FI,qCACE,uB7Bi3FN,C6Bx2FI,cAtBF,sBAuBI,W7B22FJ,C6Bx2FI,wCACE,2B7B02FN,C6Bt2FI,6BAOE,qCAAA,CACA,+CAAA,CAAA,uC7B22FN,C6Bj2FI,yDAZE,UAAA,CADA,YAAA,CAIA,4BAAA,CAAA,oBAAA,CACA,6BAAA,CAAA,qBAAA,CACA,yBAAA,CAAA,iBAAA,CAVA,iBAAA,CACA,SAAA,CAEA,WAAA,CADA,U7B+3FN,C6Bh3FI,4BAOE,oDAAA,CAMA,4CAAA,CAAA,oCAAA,CADA,uBAAA,CAJA,+C7Bw2FN,C6B71FM,gDACE,uB7B+1FR,C6B31FM,mFACE,0C7B61FR,CACF,C6Bx1FI,0CAGE,2BAAA,CADA,uBAAA,CADA,S7B41FN,C6Bt1FI,8CACE,oB7Bw1FN,C6Br1FM,aAJF,8CASI,8CAAA,CACA,iBAAA,CAHA,gCAAA,CADA,eAAA,CADA,cAAA,CAGA,kB7B01FN,C6Br1FM,oDACE,mC7Bu1FR,CACF,C6B30FE,gCAEE,iBAAA,CADA,e7B+0FJ,C6B30FI,mCACE,iB7B60FN,C6B10FM,oDAGE,a7Bw1FR,C6B31FM,oDAGE,c7Bw1FR,C6B31FM,0CAcE,8CAAA,CACA,iBAAA,CALA,gCAAA,CAEA,oBAAA,CACA,qBAAA,CANA,iBAAA,CACA,eAAA,CAHA,UAAA,CAIA,gBAAA,CALA,aAAA,CAEA,cAAA,CALA,iBAAA,CAUA,iBAAA,CATA,S7By1FR,C8BvmGA,kBAME,e9BmnGF,C8BznGA,kBAME,gB9BmnGF,C8BznGA,QAUE,2CAAA,CACA,oBAAA,CAEA,8BAAA,CALA,uCAAA,CACA,cAAA,CALA,aAAA,CAGA,eAAA,CAKA,YAAA,CAPA,mBAAA,CAJA,cAAA,CACA,UAAA,CAiBA,yBAAA,CALA,mGACE,CAZF,S9BsnGF,C8BnmGE,aAtBF,QAuBI,Y9BsmGF,CACF,C8BnmGE,kBACE,wB9BqmGJ,C8BjmGE,gBAEE,SAAA,CADA,mBAAA,CAGA,+BAAA,CADA,uB9BomGJ,C8BhmGI,0BACE,8B9BkmGN,C8B7lGE,4BAEE,0CAAA,CADA,+B9BgmGJ,C8B3lGE,YACE,oBAAA,CACA,oB9B6lGJ,C+BlpGA,oBACE,GACE,mB/BqpGF,CACF,C+B7oGA,MACE,wf/B+oGF,C+BzoGA,YACE,aAAA,CAEA,eAAA,CADA,a/B6oGF,C+BzoGE,+BAOE,kBAAA,CAAA,kB/B0oGJ,C+BjpGE,+BAOE,iBAAA,CAAA,mB/B0oGJ,C+BjpGE,qBAQE,aAAA,CACA,cAAA,CACA,YAAA,CATA,iBAAA,CAKA,U/B2oGJ,C+BpoGI,qCAIE,iB/B4oGN,C+BhpGI,qCAIE,kB/B4oGN,C+BhpGI,2BAME,6BAAA,CADA,UAAA,CAJA,oBAAA,CAEA,YAAA,CAIA,yCAAA,CAAA,iCAAA,CACA,4BAAA,CAAA,oBAAA,CACA,6BAAA,CAAA,qBAAA,CACA,yBAAA,CAAA,iBAAA,CARA,W/B8oGN,C+BjoGE,kBAUE,2CAAA,CACA,mBAAA,CACA,8BAAA,CAJA,gCAAA,CACA,oBAAA,CAHA,kBAAA,CAFA,YAAA,CASA,SAAA,CANA,aAAA,CAFA,SAAA,CAJA,iBAAA,CAgBA,4BAAA,CAfA,UAAA,CAYA,+CACE,CAZF,S/B+oGJ,C+B9nGI,+EACE,gBAAA,CACA,SAAA,CACA,sC/BgoGN,C+B1nGI,qCAEE,oCACE,gC/B2nGN,C+BvnGI,2CACE,c/BynGN,CACF,C+BpnGE,kBACE,kB/BsnGJ,C+BlnGE,4BAGE,kBAAA,CAAA,oB/BynGJ,C+B5nGE,4BAGE,mBAAA,CAAA,mB/BynGJ,C+B5nGE,kBAKE,cAAA,CAJA,aAAA,CAKA,YAAA,CAIA,uBAAA,CAHA,2CACE,CAJF,kBAAA,CAFA,U/B0nGJ,C+B/mGI,gDACE,+B/BinGN,C+B7mGI,wBACE,qD/B+mGN,CgC/sGA,MAEI,uWAAA,CAAA,8WAAA,CAAA,sPAAA,CAAA,8xBAAA,CAAA,0MAAA,CAAA,gbAAA,CAAA,gMAAA,CAAA,iQAAA,CAAA,0VAAA,CAAA,6aAAA,CAAA,8SAAA,CAAA,gMhCwuGJ,CgC5tGE,4CAME,8CAAA,CACA,4BAAA,CACA,mBAAA,CACA,8BAAA,CAJA,mCAAA,CAJA,iBAAA,CAGA,gBAAA,CADA,iBAAA,CADA,eAAA,CASA,uBAAA,CADA,2BhCguGJ,CgC5tGI,aAdF,4CAeI,ehC+tGJ,CACF,CgC5tGI,sEACE,gChC8tGN,CgCztGI,gDACE,qBhC2tGN,CgCvtGI,gIAEE,iBAAA,CADA,chC0tGN,CgCrtGI,4FACE,iBhCutGN,CgCntGI,kFACE,ehCqtGN,CgCjtGI,0FACE,YhCmtGN,CgC/sGI,8EACE,mBhCitGN,CgC5sGE,sEAGE,iBAAA,CAAA,mBhCstGJ,CgCztGE,sEAGE,kBAAA,CAAA,kBhCstGJ,CgCztGE,sEASE,uBhCgtGJ,CgCztGE,sEASE,wBhCgtGJ,CgCztGE,sEAUE,4BhC+sGJ,CgCztGE,4IAWE,6BhC8sGJ,CgCztGE,sEAWE,4BhC8sGJ,CgCztGE,kDAOE,0BAAA,CACA,WAAA,CAFA,eAAA,CADA,eAAA,CAHA,oBAAA,CAAA,iBAAA,CADA,iBhCwtGJ,CgC3sGI,kFACE,ehC6sGN,CgCzsGI,oFAOE,UhC+sGN,CgCttGI,oFAOE,WhC+sGN,CgCttGI,gEAME,wBfkIU,CenIV,UAAA,CADA,WAAA,CAIA,kDAAA,CAAA,0CAAA,CACA,4BAAA,CAAA,oBAAA,CACA,6BAAA,CAAA,qBAAA,CACA,yBAAA,CAAA,iBAAA,CAVA,iBAAA,CACA,UAAA,CACA,UhCmtGN,CgCvsGI,4DACE,4DhCysGN,CgC3rGE,sDACE,oBhC8rGJ,CgC3rGI,gFACE,gChC6rGN,CgCxrGE,8DACE,0BhC2rGJ,CgCxrGI,4EACE,wBAlBG,CAmBH,kDAAA,CAAA,0ChC0rGN,CgCtrGI,0EACE,ahCwrGN,CgC7sGE,8DACE,oBhCgtGJ,CgC7sGI,wFACE,gChC+sGN,CgC1sGE,sEACE,0BhC6sGJ,CgC1sGI,oFACE,wBAlBG,CAmBH,sDAAA,CAAA,8ChC4sGN,CgCxsGI,kFACE,ahC0sGN,CgC/tGE,sDACE,oBhCkuGJ,CgC/tGI,gFACE,gChCiuGN,CgC5tGE,8DACE,0BhC+tGJ,CgC5tGI,4EACE,wBAlBG,CAmBH,kDAAA,CAAA,0ChC8tGN,CgC1tGI,0EACE,ahC4tGN,CgCjvGE,oDACE,oBhCovGJ,CgCjvGI,8EACE,gChCmvGN,CgC9uGE,4DACE,0BhCivGJ,CgC9uGI,0EACE,wBAlBG,CAmBH,iDAAA,CAAA,yChCgvGN,CgC5uGI,wEACE,ahC8uGN,CgCnwGE,4DACE,oBhCswGJ,CgCnwGI,sFACE,gChCqwGN,CgChwGE,oEACE,0BhCmwGJ,CgChwGI,kFACE,wBAlBG,CAmBH,qDAAA,CAAA,6ChCkwGN,CgC9vGI,gFACE,ahCgwGN,CgCrxGE,8DACE,oBhCwxGJ,CgCrxGI,wFACE,gChCuxGN,CgClxGE,sEACE,0BhCqxGJ,CgClxGI,oFACE,wBAlBG,CAmBH,sDAAA,CAAA,8ChCoxGN,CgChxGI,kFACE,ahCkxGN,CgCvyGE,4DACE,oBhC0yGJ,CgCvyGI,sFACE,gChCyyGN,CgCpyGE,oEACE,0BhCuyGJ,CgCpyGI,kFACE,wBAlBG,CAmBH,qDAAA,CAAA,6ChCsyGN,CgClyGI,gFACE,ahCoyGN,CgCzzGE,4DACE,oBhC4zGJ,CgCzzGI,sFACE,gChC2zGN,CgCtzGE,oEACE,0BhCyzGJ,CgCtzGI,kFACE,wBAlBG,CAmBH,qDAAA,CAAA,6ChCwzGN,CgCpzGI,gFACE,ahCszGN,CgC30GE,0DACE,oBhC80GJ,CgC30GI,oFACE,gChC60GN,CgCx0GE,kEACE,0BhC20GJ,CgCx0GI,gFACE,wBAlBG,CAmBH,oDAAA,CAAA,4ChC00GN,CgCt0GI,8EACE,ahCw0GN,CgC71GE,oDACE,oBhCg2GJ,CgC71GI,8EACE,gChC+1GN,CgC11GE,4DACE,0BhC61GJ,CgC11GI,0EACE,wBAlBG,CAmBH,iDAAA,CAAA,yChC41GN,CgCx1GI,wEACE,ahC01GN,CgC/2GE,4DACE,oBhCk3GJ,CgC/2GI,sFACE,gChCi3GN,CgC52GE,oEACE,0BhC+2GJ,CgC52GI,kFACE,wBAlBG,CAmBH,qDAAA,CAAA,6ChC82GN,CgC12GI,gFACE,ahC42GN,CgCj4GE,wDACE,oBhCo4GJ,CgCj4GI,kFACE,gChCm4GN,CgC93GE,gEACE,0BhCi4GJ,CgC93GI,8EACE,wBAlBG,CAmBH,mDAAA,CAAA,2ChCg4GN,CgC53GI,4EACE,ahC83GN,CiCliHA,MACE,wMjCqiHF,CiC5hHE,sBAEE,uCAAA,CADA,gBjCgiHJ,CiC5hHI,mCACE,ajC8hHN,CiC/hHI,mCACE,cjC8hHN,CiC1hHM,4BACE,sBjC4hHR,CiCzhHQ,mCACE,gCjC2hHV,CiCvhHQ,2DACE,SAAA,CAEA,uBAAA,CADA,ejC0hHV,CiCrhHQ,yGACE,SAAA,CACA,uBjCuhHV,CiCnhHQ,yCACE,YjCqhHV,CiC9gHE,0BACE,eAAA,CACA,ejCghHJ,CiC7gHI,+BACE,oBjC+gHN,CiC1gHE,gDACE,YjC4gHJ,CiCxgHE,8BAIE,+BAAA,CAHA,oBAAA,CAEA,WAAA,CAGA,SAAA,CAKA,4BAAA,CAJA,4DACE,CAHF,0BjC4gHJ,CiCngHI,aAdF,8BAeI,+BAAA,CACA,SAAA,CACA,uBjCsgHJ,CACF,CiCngHI,wCACE,6BjCqgHN,CiCjgHI,oCACE,+BjCmgHN,CiC//GI,qCAKE,6BAAA,CADA,UAAA,CAHA,oBAAA,CAEA,YAAA,CAGA,2CAAA,CAAA,mCAAA,CACA,4BAAA,CAAA,oBAAA,CACA,6BAAA,CAAA,qBAAA,CACA,yBAAA,CAAA,iBAAA,CAPA,WjCwgHN,CiC3/GQ,mDACE,oBjC6/GV,CkC3mHE,kCAEE,iBlCinHJ,CkCnnHE,kCAEE,kBlCinHJ,CkCnnHE,wBAGE,yCAAA,CAFA,oBAAA,CAGA,SAAA,CACA,mClC8mHJ,CkCzmHI,aAVF,wBAWI,YlC4mHJ,CACF,CkCxmHE,6FAEE,SAAA,CACA,mClC0mHJ,CkCpmHE,4FAEE,+BlCsmHJ,CkClmHE,oBACE,yBAAA,CACA,uBAAA,CAGA,yElCkmHJ,CKn+GI,sC6BrHE,qDACE,uBlC2lHN,CACF,CkCtlHE,kEACE,yBlCwlHJ,CkCplHE,sBACE,0BlCslHJ,CmCjpHE,2BACE,anCopHJ,CK/9GI,0C8BtLF,2BAKI,enCopHJ,CmCjpHI,6BACE,yBAAA,CAAA,iBnCmpHN,CACF,CmC/oHI,6BAEE,0BAAA,CAAA,2BAAA,CADA,eAAA,CAEA,iBnCipHN,CmC9oHM,2CACE,kBnCgpHR,CmC1oHI,6CACE,QnC4oHN,CoCxqHE,uBACE,4CpC4qHJ,CoCvqHE,8CAJE,kCAAA,CAAA,0BpC+qHJ,CoC3qHE,uBACE,4CpC0qHJ,CoCrqHE,4BAEE,kCAAA,CAAA,0BAAA,CADA,qCpCwqHJ,CoCpqHI,mCACE,apCsqHN,CoClqHI,kCACE,apCoqHN,CoC/pHE,0BAKE,eAAA,CAJA,aAAA,CAEA,YAAA,CACA,aAAA,CAFA,kBAAA,CAAA,mBpCoqHJ,CoC9pHI,uCACE,epCgqHN,CoC5pHI,sCACE,kBpC8pHN,CqC3sHA,MACE,8LrC8sHF,CqCrsHE,oBAGE,iBAAA,CAEA,gBAAA,CADA,arCusHJ,CqCnsHI,wCACE,uBrCqsHN,CqCjsHI,gCAEE,eAAA,CADA,gBrCosHN,CqC7rHM,wCACE,mBrC+rHR,CqCzrHE,8BAKE,oBrC6rHJ,CqClsHE,8BAKE,mBrC6rHJ,CqClsHE,8BAUE,4BrCwrHJ,CqClsHE,4DAWE,6BrCurHJ,CqClsHE,8BAWE,4BrCurHJ,CqClsHE,oBASE,cAAA,CANA,aAAA,CACA,eAAA,CAIA,erC0rHJ,CqCprHI,kCACE,uCAAA,CACA,oBrCsrHN,CqClrHI,wCAEE,uCAAA,CADA,YrCqrHN,CqChrHI,oCASE,WrCsrHN,CqC/rHI,oCASE,UrCsrHN,CqC/rHI,0BAME,6BAAA,CADA,UAAA,CADA,WAAA,CAMA,yCAAA,CAAA,iCAAA,CACA,4BAAA,CAAA,oBAAA,CACA,6BAAA,CAAA,qBAAA,CACA,yBAAA,CAAA,iBAAA,CAZA,iBAAA,CACA,UAAA,CAMA,sBAAA,CADA,yBAAA,CAJA,UrC4rHN,CqC/qHM,oCACE,wBrCirHR,CqC5qHI,4BACE,YrC8qHN,CqCzqHI,4CACE,YrC2qHN,CsCrwHE,+DACE,sBAAA,CAEA,mBAAA,CACA,0BAAA,CACA,uBtCuwHJ,CsCpwHI,2EAGE,iBAAA,CADA,eAAA,CADA,yBtCwwHN,CsCjwHE,mEACE,0BtCmwHJ,CsC/vHE,oBACE,qBtCiwHJ,CsC7vHE,gBACE,oBtC+vHJ,CsC3vHE,gBACE,qBtC6vHJ,CsCzvHE,iBACE,kBtC2vHJ,CsCvvHE,kBACE,kBtCyvHJ,CuClyHE,6BACE,sCvCqyHJ,CuClyHE,cACE,yCvCoyHJ,CuCxxHE,sIACE,oCvC0xHJ,CuClxHE,2EACE,qCvCoxHJ,CuC1wHE,wGACE,oCvC4wHJ,CuCnwHE,yFACE,qCvCqwHJ,CuChwHE,6BACE,kCvCkwHJ,CuC5vHE,6CACE,sCvC8vHJ,CuCvvHE,4DACE,sCvCyvHJ,CuClvHE,4DACE,qCvCovHJ,CuC3uHE,yFACE,qCvC6uHJ,CuCruHE,2EACE,sCvCuuHJ,CuC5tHE,wHACE,qCvC8tHJ,CuCztHE,8BAGE,mBAAA,CADA,gBAAA,CADA,gBvC6tHJ,CuCxtHE,eACE,4CvC0tHJ,CuCvtHE,eACE,4CvCytHJ,CuCrtHE,gBAIE,+CAAA,CACA,kDAAA,CAJA,aAAA,CAEA,wBAAA,CADA,wBvC0tHJ,CuCntHE,yBAOE,wCAAA,CACA,+DAAA,CACA,4BAAA,CACA,6BAAA,CARA,iBAAA,CAGA,eAAA,CACA,eAAA,CAFA,cAAA,CADA,oCAAA,CAFA,iBvC8tHJ,CuCltHI,6BACE,YvCotHN,CuCjtHM,kCACE,wBAAA,CACA,yBvCmtHR,CuC7sHE,iCAaE,wCAAA,CACA,+DAAA,CAJA,uCAAA,CACA,0BAAA,CALA,UAAA,CAJA,oBAAA,CAOA,2BAAA,CADA,2BAAA,CADA,2BAAA,CANA,eAAA,CAWA,wBAAA,CAAA,gBAAA,CAPA,SvCstHJ,CuCpsHE,sBACE,iBAAA,CACA,iBvCssHJ,CuC9rHI,sCACE,gBvCgsHN,CuC5rHI,gDACE,YvC8rHN,CuCprHA,gBACE,iBvCurHF,CuCnrHE,yCACE,aAAA,CACA,SvCqrHJ,CuChrHE,mBACE,YvCkrHJ,CuC7qHE,oBACE,QvC+qHJ,CuC3qHE,4BACE,WAAA,CACA,SAAA,CACA,evC6qHJ,CuC1qHI,0CACE,YvC4qHN,CuCtqHE,yBAKE,wCAAA,CAEA,+BAAA,CADA,4BAAA,CAHA,eAAA,CADA,oDAAA,CAEA,wBAAA,CAAA,gBvC2qHJ,CuCpqHE,2BAEE,+DAAA,CADA,2BvCuqHJ,CuCnqHI,+BACE,uCAAA,CACA,gBvCqqHN,CuChqHE,sBACE,MAAA,CACA,WvCkqHJ,CuC7pHA,aACE,avCgqHF,CuCtpHE,4BAEE,aAAA,CADA,YvC0pHJ,CuCtpHI,wDAEE,2BAAA,CADA,wBvCypHN,CuCnpHE,+BAKE,2CAAA,CAEA,+BAAA,CADA,gCAAA,CADA,sBAAA,CAHA,mBAAA,CACA,gBAAA,CAFA,avC2pHJ,CuClpHI,qCAEE,UAAA,CACA,UAAA,CAFA,avCspHN,CKxxHI,0CkCiJF,8BACE,iBvC2oHF,CuCjoHE,wSAGE,evCuoHJ,CuCnoHE,sCAEE,mBAAA,CACA,eAAA,CADA,oBAAA,CADA,kBAAA,CAAA,mBvCuoHJ,CACF,CwC/9HI,yDAIE,+BAAA,CACA,8BAAA,CAFA,aAAA,CADA,QAAA,CADA,iBxCq+HN,CwC79HI,uBAEE,uCAAA,CADA,cxCg+HN,CwC36HM,iHAEE,WAlDkB,CAiDlB,kBxCs7HR,CwCv7HM,6HAEE,WAlDkB,CAiDlB,kBxCk8HR,CwCn8HM,6HAEE,WAlDkB,CAiDlB,kBxC88HR,CwC/8HM,oHAEE,WAlDkB,CAiDlB,kBxC09HR,CwC39HM,0HAEE,WAlDkB,CAiDlB,kBxCs+HR,CwCv+HM,uHAEE,WAlDkB,CAiDlB,kBxCk/HR,CwCn/HM,uHAEE,WAlDkB,CAiDlB,kBxC8/HR,CwC//HM,6HAEE,WAlDkB,CAiDlB,kBxC0gIR,CwC3gIM,yCAEE,WAlDkB,CAiDlB,kBxC8gIR,CwC/gIM,yCAEE,WAlDkB,CAiDlB,kBxCkhIR,CwCnhIM,0CAEE,WAlDkB,CAiDlB,kBxCshIR,CwCvhIM,uCAEE,WAlDkB,CAiDlB,kBxC0hIR,CwC3hIM,wCAEE,WAlDkB,CAiDlB,kBxC8hIR,CwC/hIM,sCAEE,WAlDkB,CAiDlB,kBxCkiIR,CwCniIM,wCAEE,WAlDkB,CAiDlB,kBxCsiIR,CwCviIM,oCAEE,WAlDkB,CAiDlB,kBxC0iIR,CwC3iIM,2CAEE,WAlDkB,CAiDlB,kBxC8iIR,CwC/iIM,qCAEE,WAlDkB,CAiDlB,kBxCkjIR,CwCnjIM,oCAEE,WAlDkB,CAiDlB,kBxCsjIR,CwCvjIM,kCAEE,WAlDkB,CAiDlB,kBxC0jIR,CwC3jIM,qCAEE,WAlDkB,CAiDlB,kBxC8jIR,CwC/jIM,mCAEE,WAlDkB,CAiDlB,kBxCkkIR,CwCnkIM,qCAEE,WAlDkB,CAiDlB,kBxCskIR,CwCvkIM,wCAEE,WAlDkB,CAiDlB,kBxC0kIR,CwC3kIM,sCAEE,WAlDkB,CAiDlB,kBxC8kIR,CwC/kIM,2CAEE,WAlDkB,CAiDlB,kBxCklIR,CwCvkIM,iCAEE,WAPkB,CAMlB,iBxC0kIR,CwC3kIM,uCAEE,WAPkB,CAMlB,iBxC8kIR,CwC/kIM,mCAEE,WAPkB,CAMlB,iBxCklIR,CyCpqIA,MACE,qMAAA,CACA,mMzCuqIF,CyC9pIE,wBAKE,mBAAA,CAHA,YAAA,CACA,qBAAA,CACA,YAAA,CAHA,iBzCqqIJ,CyC3pII,8BAGE,QAAA,CACA,SAAA,CAHA,iBAAA,CACA,OzC+pIN,CyC1pIM,qCACE,0BzC4pIR,CyC/nIM,kEACE,0CzCioIR,CyC3nIE,2BAKE,uBAAA,CADA,+DAAA,CAHA,YAAA,CACA,cAAA,CACA,aAAA,CAGA,oBzC6nIJ,CyC1nII,aATF,2BAUI,gBzC6nIJ,CACF,CyC1nII,cAGE,+BACE,iBzC0nIN,CyCvnIM,sCAQE,qCAAA,CANA,QAAA,CAKA,UAAA,CAHA,aAAA,CAEA,UAAA,CAHA,MAAA,CAFA,iBAAA,CAaA,2CAAA,CALA,2DACE,CAGF,kDAAA,CARA,+BzC+nIR,CACF,CyCjnII,8CACE,YzCmnIN,CyC/mII,iCASE,+BAAA,CACA,6BAAA,CAJA,uCAAA,CAEA,cAAA,CAPA,aAAA,CAGA,gBAAA,CACA,eAAA,CAFA,8BAAA,CAWA,+BAAA,CAHA,2CACE,CALF,kBAAA,CALA,UzC2nIN,CyC5mIM,aAII,6CACE,OzC2mIV,CyC5mIQ,8CACE,OzC8mIV,CyC/mIQ,8CACE,OzCinIV,CyClnIQ,8CACE,OzConIV,CyCrnIQ,8CACE,OzCunIV,CyCxnIQ,8CACE,OzC0nIV,CyC3nIQ,8CACE,OzC6nIV,CyC9nIQ,8CACE,OzCgoIV,CyCjoIQ,8CACE,OzCmoIV,CyCpoIQ,+CACE,QzCsoIV,CyCvoIQ,+CACE,QzCyoIV,CyC1oIQ,+CACE,QzC4oIV,CyC7oIQ,+CACE,QzC+oIV,CyChpIQ,+CACE,QzCkpIV,CyCnpIQ,+CACE,QzCqpIV,CyCtpIQ,+CACE,QzCwpIV,CyCzpIQ,+CACE,QzC2pIV,CyC5pIQ,+CACE,QzC8pIV,CyC/pIQ,+CACE,QzCiqIV,CyClqIQ,+CACE,QzCoqIV,CACF,CyC/pIM,uCACE,gCzCiqIR,CyC7pIM,oDACE,azC+pIR,CyC1pII,yCACE,SzC4pIN,CyCxpIM,2CACE,aAAA,CACA,8BzC0pIR,CyCppIE,4BACE,UzCspIJ,CyCnpII,aAJF,4BAKI,gBzCspIJ,CACF,CyClpIE,0BACE,YzCopIJ,CyCjpII,aAJF,0BAKI,azCopIJ,CyChpIM,sCACE,OzCkpIR,CyCnpIM,uCACE,OzCqpIR,CyCtpIM,uCACE,OzCwpIR,CyCzpIM,uCACE,OzC2pIR,CyC5pIM,uCACE,OzC8pIR,CyC/pIM,uCACE,OzCiqIR,CyClqIM,uCACE,OzCoqIR,CyCrqIM,uCACE,OzCuqIR,CyCxqIM,uCACE,OzC0qIR,CyC3qIM,wCACE,QzC6qIR,CyC9qIM,wCACE,QzCgrIR,CyCjrIM,wCACE,QzCmrIR,CyCprIM,wCACE,QzCsrIR,CyCvrIM,wCACE,QzCyrIR,CyC1rIM,wCACE,QzC4rIR,CyC7rIM,wCACE,QzC+rIR,CyChsIM,wCACE,QzCksIR,CyCnsIM,wCACE,QzCqsIR,CyCtsIM,wCACE,QzCwsIR,CyCzsIM,wCACE,QzC2sIR,CACF,CyCrsII,+FAEE,QzCusIN,CyCpsIM,yGACE,wBAAA,CACA,yBzCusIR,CyC9rIM,2DAEE,wBAAA,CACA,yBAAA,CAFA,QzCksIR,CyC3rIM,iEACE,QzC6rIR,CyC1rIQ,qLAGE,wBAAA,CACA,yBAAA,CAFA,QzC8rIV,CyCxrIQ,6FACE,wBAAA,CACA,yBzC0rIV,CyCrrIM,yDACE,kBzCurIR,CyClrII,sCACE,QzCorIN,CyC/qIE,2BAEE,iBAAA,CAOA,kBAAA,CAHA,uCAAA,CAEA,cAAA,CAPA,aAAA,CAGA,YAAA,CACA,gBAAA,CAEA,mBAAA,CAGA,gCAAA,CAPA,WzCwrIJ,CyC9qII,iCAEE,uDAAA,CADA,+BzCirIN,CyC5qII,iCAKE,6BAAA,CADA,UAAA,CAHA,aAAA,CAEA,WAAA,CAMA,8CAAA,CAAA,sCAAA,CACA,4BAAA,CAAA,oBAAA,CACA,6BAAA,CAAA,qBAAA,CACA,yBAAA,CAAA,iBAAA,CANA,+CACE,CALF,UzCsrIN,CyCvqIE,4BAOE,yEACE,CANF,YAAA,CAGA,aAAA,CAFA,qBAAA,CAGA,mBAAA,CALA,iBAAA,CAYA,wBAAA,CATA,YzC6qIJ,CyCjqII,sCACE,wBzCmqIN,CyC/pII,oCACE,SzCiqIN,CyC7pII,kCAGE,wEACE,CAFF,mBAAA,CADA,OzCiqIN,CyCvpIM,uDACE,8CAAA,CAAA,sCzCypIR,CKhyII,0CoCqJF,wDAEE,kBzCipIF,CyCnpIA,wDAEE,mBzCipIF,CyCnpIA,8CAGE,eAAA,CAFA,eAAA,CAGA,iCzC+oIF,CyC3oIE,8DACE,mBzC8oIJ,CyC/oIE,8DACE,kBzC8oIJ,CyC/oIE,oDAEE,UzC6oIJ,CyCzoIE,8EAEE,kBzC4oIJ,CyC9oIE,8EAEE,mBzC4oIJ,CyC9oIE,8EAGE,kBzC2oIJ,CyC9oIE,8EAGE,mBzC2oIJ,CyC9oIE,oEACE,UzC6oIJ,CyCvoIE,8EAEE,mBzC0oIJ,CyC5oIE,8EAEE,kBzC0oIJ,CyC5oIE,8EAGE,mBzCyoIJ,CyC5oIE,8EAGE,kBzCyoIJ,CyC5oIE,oEACE,UzC2oIJ,CACF,CyC7nIE,cAHF,olDAII,gCzCgoIF,CyC7nIE,g8GACE,uCzC+nIJ,CACF,CyC1nIA,4sDACE,+BzC6nIF,CyCznIA,wmDACE,azC4nIF,C0ChgJA,MACE,8WAAA,CACA,uX1CmgJF,C0C1/IE,4BAEE,oBAAA,CADA,iB1C8/IJ,C0Cz/II,sDAGE,S1C2/IN,C0C9/II,sDAGE,U1C2/IN,C0C9/II,4CACE,iBAAA,CACA,S1C4/IN,C0Ct/IE,+CAEE,SAAA,CADA,U1Cy/IJ,C0Cp/IE,kDAOE,W1C0/IJ,C0CjgJE,kDAOE,Y1C0/IJ,C0CjgJE,wCAME,qDAAA,CADA,UAAA,CADA,aAAA,CAIA,0CAAA,CAAA,kCAAA,CACA,4BAAA,CAAA,oBAAA,CACA,6BAAA,CAAA,qBAAA,CACA,yBAAA,CAAA,iBAAA,CAVA,iBAAA,CACA,SAAA,CACA,Y1C8/IJ,C0Cl/IE,gEACE,wBzB2Wa,CyB1Wb,mDAAA,CAAA,2C1Co/IJ,C2CpiJA,QACE,8DAAA,CAGA,+CAAA,CACA,iEAAA,CACA,oDAAA,CACA,sDAAA,CACA,mDAAA,CAGA,qEAAA,CACA,qEAAA,CACA,wEAAA,CACA,0EAAA,CACA,wEAAA,CACA,yEAAA,CACA,kEAAA,CACA,+DAAA,CACA,oEAAA,CACA,oEAAA,CACA,mEAAA,CACA,gEAAA,CACA,uEAAA,CACA,mEAAA,CACA,qEAAA,CACA,oEAAA,CACA,gEAAA,CACA,wEAAA,CACA,qEAAA,CACA,+D3CmiJF,C2C7hJA,SAEE,kBAAA,CADA,Y3CiiJF,C4CnkJE,kBAUE,cAAA,CATA,YAAA,CACA,kEACE,CAQF,Y5C+jJJ,C4C3jJI,sDACE,gB5C6jJN,C4CvjJI,oFAKE,wDAAA,CACA,mBAAA,CAJA,aAAA,CAEA,QAAA,CADA,aAAA,CAIA,sC5CyjJN,C4CpjJM,iOACE,kBAAA,CACA,8B5CujJR,C4CnjJM,6FACE,iBAAA,CAAA,c5CsjJR,C4CljJM,2HACE,Y5CqjJR,C4CjjJM,wHACE,e5CojJR,C4CriJI,yMAGE,eAAA,CAAA,Y5C6iJN,C4C/hJI,ybAOE,W5CqiJN,C4CjiJI,8BACE,eAAA,CAAA,Y5CmiJN,CK/9II,mCwChKA,8BACE,U7CuoJJ,C6CxoJE,8BACE,W7CuoJJ,C6CxoJE,8BAGE,kB7CqoJJ,C6CxoJE,8BAGE,iB7CqoJJ,C6CxoJE,oBAKE,mBAAA,CADA,YAAA,CAFA,a7CsoJJ,C6ChoJI,kCACE,W7CmoJN,C6CpoJI,kCACE,U7CmoJN,C6CpoJI,kCAEE,iBAAA,CAAA,c7CkoJN,C6CpoJI,kCAEE,aAAA,CAAA,kB7CkoJN,CACF","file":"main.css"}
\ No newline at end of file
diff --git a/assets/stylesheets/palette.06af60db.min.css b/assets/stylesheets/palette.06af60db.min.css
new file mode 100644
index 000000000..a640d38ab
--- /dev/null
+++ b/assets/stylesheets/palette.06af60db.min.css
@@ -0,0 +1 @@
+@media screen{[data-md-color-scheme=slate]{--md-default-fg-color:hsla(var(--md-hue),15%,90%,0.82);--md-default-fg-color--light:hsla(var(--md-hue),15%,90%,0.56);--md-default-fg-color--lighter:hsla(var(--md-hue),15%,90%,0.32);--md-default-fg-color--lightest:hsla(var(--md-hue),15%,90%,0.12);--md-default-bg-color:hsla(var(--md-hue),15%,14%,1);--md-default-bg-color--light:hsla(var(--md-hue),15%,14%,0.54);--md-default-bg-color--lighter:hsla(var(--md-hue),15%,14%,0.26);--md-default-bg-color--lightest:hsla(var(--md-hue),15%,14%,0.07);--md-code-fg-color:hsla(var(--md-hue),18%,86%,0.82);--md-code-bg-color:hsla(var(--md-hue),15%,18%,1);--md-code-hl-color:#2977ff;--md-code-hl-color--light:#2977ff1a;--md-code-hl-number-color:#e6695b;--md-code-hl-special-color:#f06090;--md-code-hl-function-color:#c973d9;--md-code-hl-constant-color:#9383e2;--md-code-hl-keyword-color:#6791e0;--md-code-hl-string-color:#2fb170;--md-code-hl-name-color:var(--md-code-fg-color);--md-code-hl-operator-color:var(--md-default-fg-color--light);--md-code-hl-punctuation-color:var(--md-default-fg-color--light);--md-code-hl-comment-color:var(--md-default-fg-color--light);--md-code-hl-generic-color:var(--md-default-fg-color--light);--md-code-hl-variable-color:var(--md-default-fg-color--light);--md-typeset-color:var(--md-default-fg-color);--md-typeset-a-color:var(--md-primary-fg-color);--md-typeset-kbd-color:hsla(var(--md-hue),15%,90%,0.12);--md-typeset-kbd-accent-color:hsla(var(--md-hue),15%,90%,0.2);--md-typeset-kbd-border-color:hsla(var(--md-hue),15%,14%,1);--md-typeset-mark-color:#4287ff4d;--md-typeset-table-color:hsla(var(--md-hue),15%,95%,0.12);--md-typeset-table-color--light:hsla(var(--md-hue),15%,95%,0.035);--md-admonition-fg-color:var(--md-default-fg-color);--md-admonition-bg-color:var(--md-default-bg-color);--md-footer-bg-color:hsla(var(--md-hue),15%,10%,0.87);--md-footer-bg-color--dark:hsla(var(--md-hue),15%,8%,1);--md-shadow-z1:0 0.2rem 0.5rem #0000000d,0 0 0.05rem #0000001a;--md-shadow-z2:0 0.2rem 0.5rem #00000040,0 0 0.05rem #00000040;--md-shadow-z3:0 0.2rem 0.5rem #0006,0 0 0.05rem #00000059;color-scheme:dark}[data-md-color-scheme=slate] img[src$="#gh-light-mode-only"],[data-md-color-scheme=slate] img[src$="#only-light"]{display:none}[data-md-color-scheme=slate][data-md-color-primary=pink]{--md-typeset-a-color:#ed5487}[data-md-color-scheme=slate][data-md-color-primary=purple]{--md-typeset-a-color:#c46fd3}[data-md-color-scheme=slate][data-md-color-primary=deep-purple]{--md-typeset-a-color:#a47bea}[data-md-color-scheme=slate][data-md-color-primary=indigo]{--md-typeset-a-color:#5488e8}[data-md-color-scheme=slate][data-md-color-primary=teal]{--md-typeset-a-color:#00ccb8}[data-md-color-scheme=slate][data-md-color-primary=green]{--md-typeset-a-color:#71c174}[data-md-color-scheme=slate][data-md-color-primary=deep-orange]{--md-typeset-a-color:#ff764d}[data-md-color-scheme=slate][data-md-color-primary=brown]{--md-typeset-a-color:#c1775c}[data-md-color-scheme=slate][data-md-color-primary=black],[data-md-color-scheme=slate][data-md-color-primary=blue-grey],[data-md-color-scheme=slate][data-md-color-primary=grey],[data-md-color-scheme=slate][data-md-color-primary=white]{--md-typeset-a-color:#5e8bde}[data-md-color-switching] *,[data-md-color-switching] :after,[data-md-color-switching] :before{transition-duration:0ms!important}}[data-md-color-accent=red]{--md-accent-fg-color:#ff1947;--md-accent-fg-color--transparent:#ff19471a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-accent=pink]{--md-accent-fg-color:#f50056;--md-accent-fg-color--transparent:#f500561a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-accent=purple]{--md-accent-fg-color:#df41fb;--md-accent-fg-color--transparent:#df41fb1a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-accent=deep-purple]{--md-accent-fg-color:#7c4dff;--md-accent-fg-color--transparent:#7c4dff1a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-accent=indigo]{--md-accent-fg-color:#526cfe;--md-accent-fg-color--transparent:#526cfe1a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-accent=blue]{--md-accent-fg-color:#4287ff;--md-accent-fg-color--transparent:#4287ff1a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-accent=light-blue]{--md-accent-fg-color:#0091eb;--md-accent-fg-color--transparent:#0091eb1a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-accent=cyan]{--md-accent-fg-color:#00bad6;--md-accent-fg-color--transparent:#00bad61a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-accent=teal]{--md-accent-fg-color:#00bda4;--md-accent-fg-color--transparent:#00bda41a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-accent=green]{--md-accent-fg-color:#00c753;--md-accent-fg-color--transparent:#00c7531a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-accent=light-green]{--md-accent-fg-color:#63de17;--md-accent-fg-color--transparent:#63de171a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-accent=lime]{--md-accent-fg-color:#b0eb00;--md-accent-fg-color--transparent:#b0eb001a;--md-accent-bg-color:#000000de;--md-accent-bg-color--light:#0000008a}[data-md-color-accent=yellow]{--md-accent-fg-color:#ffd500;--md-accent-fg-color--transparent:#ffd5001a;--md-accent-bg-color:#000000de;--md-accent-bg-color--light:#0000008a}[data-md-color-accent=amber]{--md-accent-fg-color:#fa0;--md-accent-fg-color--transparent:#ffaa001a;--md-accent-bg-color:#000000de;--md-accent-bg-color--light:#0000008a}[data-md-color-accent=orange]{--md-accent-fg-color:#ff9100;--md-accent-fg-color--transparent:#ff91001a;--md-accent-bg-color:#000000de;--md-accent-bg-color--light:#0000008a}[data-md-color-accent=deep-orange]{--md-accent-fg-color:#ff6e42;--md-accent-fg-color--transparent:#ff6e421a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-primary=red]{--md-primary-fg-color:#ef5552;--md-primary-fg-color--light:#e57171;--md-primary-fg-color--dark:#e53734;--md-primary-bg-color:#fff;--md-primary-bg-color--light:#ffffffb3}[data-md-color-primary=pink]{--md-primary-fg-color:#e92063;--md-primary-fg-color--light:#ec417a;--md-primary-fg-color--dark:#c3185d;--md-primary-bg-color:#fff;--md-primary-bg-color--light:#ffffffb3}[data-md-color-primary=purple]{--md-primary-fg-color:#ab47bd;--md-primary-fg-color--light:#bb69c9;--md-primary-fg-color--dark:#8c24a8;--md-primary-bg-color:#fff;--md-primary-bg-color--light:#ffffffb3}[data-md-color-primary=deep-purple]{--md-primary-fg-color:#7e56c2;--md-primary-fg-color--light:#9574cd;--md-primary-fg-color--dark:#673ab6;--md-primary-bg-color:#fff;--md-primary-bg-color--light:#ffffffb3}[data-md-color-primary=indigo]{--md-primary-fg-color:#4051b5;--md-primary-fg-color--light:#5d6cc0;--md-primary-fg-color--dark:#303fa1;--md-primary-bg-color:#fff;--md-primary-bg-color--light:#ffffffb3}[data-md-color-primary=blue]{--md-primary-fg-color:#2094f3;--md-primary-fg-color--light:#42a5f5;--md-primary-fg-color--dark:#1975d2;--md-primary-bg-color:#fff;--md-primary-bg-color--light:#ffffffb3}[data-md-color-primary=light-blue]{--md-primary-fg-color:#02a6f2;--md-primary-fg-color--light:#28b5f6;--md-primary-fg-color--dark:#0287cf;--md-primary-bg-color:#fff;--md-primary-bg-color--light:#ffffffb3}[data-md-color-primary=cyan]{--md-primary-fg-color:#00bdd6;--md-primary-fg-color--light:#25c5da;--md-primary-fg-color--dark:#0097a8;--md-primary-bg-color:#fff;--md-primary-bg-color--light:#ffffffb3}[data-md-color-primary=teal]{--md-primary-fg-color:#009485;--md-primary-fg-color--light:#26a699;--md-primary-fg-color--dark:#007a6c;--md-primary-bg-color:#fff;--md-primary-bg-color--light:#ffffffb3}[data-md-color-primary=green]{--md-primary-fg-color:#4cae4f;--md-primary-fg-color--light:#68bb6c;--md-primary-fg-color--dark:#398e3d;--md-primary-bg-color:#fff;--md-primary-bg-color--light:#ffffffb3}[data-md-color-primary=light-green]{--md-primary-fg-color:#8bc34b;--md-primary-fg-color--light:#9ccc66;--md-primary-fg-color--dark:#689f38;--md-primary-bg-color:#fff;--md-primary-bg-color--light:#ffffffb3}[data-md-color-primary=lime]{--md-primary-fg-color:#cbdc38;--md-primary-fg-color--light:#d3e156;--md-primary-fg-color--dark:#b0b52c;--md-primary-bg-color:#000000de;--md-primary-bg-color--light:#0000008a}[data-md-color-primary=yellow]{--md-primary-fg-color:#ffec3d;--md-primary-fg-color--light:#ffee57;--md-primary-fg-color--dark:#fbc02d;--md-primary-bg-color:#000000de;--md-primary-bg-color--light:#0000008a}[data-md-color-primary=amber]{--md-primary-fg-color:#ffc105;--md-primary-fg-color--light:#ffc929;--md-primary-fg-color--dark:#ffa200;--md-primary-bg-color:#000000de;--md-primary-bg-color--light:#0000008a}[data-md-color-primary=orange]{--md-primary-fg-color:#ffa724;--md-primary-fg-color--light:#ffa724;--md-primary-fg-color--dark:#fa8900;--md-primary-bg-color:#000000de;--md-primary-bg-color--light:#0000008a}[data-md-color-primary=deep-orange]{--md-primary-fg-color:#ff6e42;--md-primary-fg-color--light:#ff8a66;--md-primary-fg-color--dark:#f4511f;--md-primary-bg-color:#fff;--md-primary-bg-color--light:#ffffffb3}[data-md-color-primary=brown]{--md-primary-fg-color:#795649;--md-primary-fg-color--light:#8d6e62;--md-primary-fg-color--dark:#5d4037;--md-primary-bg-color:#fff;--md-primary-bg-color--light:#ffffffb3}[data-md-color-primary=grey]{--md-primary-fg-color:#757575;--md-primary-fg-color--light:#9e9e9e;--md-primary-fg-color--dark:#616161;--md-primary-bg-color:#fff;--md-primary-bg-color--light:#ffffffb3;--md-typeset-a-color:#4051b5}[data-md-color-primary=blue-grey]{--md-primary-fg-color:#546d78;--md-primary-fg-color--light:#607c8a;--md-primary-fg-color--dark:#455a63;--md-primary-bg-color:#fff;--md-primary-bg-color--light:#ffffffb3;--md-typeset-a-color:#4051b5}[data-md-color-primary=light-green]:not([data-md-color-scheme=slate]){--md-typeset-a-color:#72ad2e}[data-md-color-primary=lime]:not([data-md-color-scheme=slate]){--md-typeset-a-color:#8b990a}[data-md-color-primary=yellow]:not([data-md-color-scheme=slate]){--md-typeset-a-color:#b8a500}[data-md-color-primary=amber]:not([data-md-color-scheme=slate]){--md-typeset-a-color:#d19d00}[data-md-color-primary=orange]:not([data-md-color-scheme=slate]){--md-typeset-a-color:#e68a00}[data-md-color-primary=white]{--md-primary-fg-color:hsla(var(--md-hue),0%,100%,1);--md-primary-fg-color--light:hsla(var(--md-hue),0%,100%,0.7);--md-primary-fg-color--dark:hsla(var(--md-hue),0%,0%,0.07);--md-primary-bg-color:hsla(var(--md-hue),0%,0%,0.87);--md-primary-bg-color--light:hsla(var(--md-hue),0%,0%,0.54);--md-typeset-a-color:#4051b5}[data-md-color-primary=white] .md-button{color:var(--md-typeset-a-color)}[data-md-color-primary=white] .md-button--primary{background-color:var(--md-typeset-a-color);border-color:var(--md-typeset-a-color);color:hsla(var(--md-hue),0%,100%,1)}@media screen and (min-width:60em){[data-md-color-primary=white] .md-search__form{background-color:hsla(var(--md-hue),0%,0%,.07)}[data-md-color-primary=white] .md-search__form:hover{background-color:hsla(var(--md-hue),0%,0%,.32)}[data-md-color-primary=white] .md-search__input+.md-search__icon{color:hsla(var(--md-hue),0%,0%,.87)}}@media screen and (min-width:76.25em){[data-md-color-primary=white] .md-tabs{border-bottom:.05rem solid #00000012}}[data-md-color-primary=black]{--md-primary-fg-color:hsla(var(--md-hue),15%,9%,1);--md-primary-fg-color--light:hsla(var(--md-hue),15%,9%,0.54);--md-primary-fg-color--dark:hsla(var(--md-hue),15%,9%,1);--md-primary-bg-color:hsla(var(--md-hue),15%,100%,1);--md-primary-bg-color--light:hsla(var(--md-hue),15%,100%,0.7);--md-typeset-a-color:#4051b5}[data-md-color-primary=black] .md-button{color:var(--md-typeset-a-color)}[data-md-color-primary=black] .md-button--primary{background-color:var(--md-typeset-a-color);border-color:var(--md-typeset-a-color);color:hsla(var(--md-hue),0%,100%,1)}[data-md-color-primary=black] .md-header{background-color:hsla(var(--md-hue),15%,9%,1)}@media screen and (max-width:59.984375em){[data-md-color-primary=black] .md-nav__source{background-color:hsla(var(--md-hue),15%,11%,.87)}}@media screen and (max-width:76.234375em){html [data-md-color-primary=black] .md-nav--primary .md-nav__title[for=__drawer]{background-color:hsla(var(--md-hue),15%,9%,1)}}@media screen and (min-width:76.25em){[data-md-color-primary=black] .md-tabs{background-color:hsla(var(--md-hue),15%,9%,1)}}
\ No newline at end of file
diff --git a/assets/stylesheets/palette.06af60db.min.css.map b/assets/stylesheets/palette.06af60db.min.css.map
new file mode 100644
index 000000000..efb568c52
--- /dev/null
+++ b/assets/stylesheets/palette.06af60db.min.css.map
@@ -0,0 +1 @@
+{"version":3,"sources":["src/templates/assets/stylesheets/palette/_scheme.scss","../../../../src/templates/assets/stylesheets/palette.scss","src/templates/assets/stylesheets/palette/_accent.scss","src/templates/assets/stylesheets/palette/_primary.scss","src/templates/assets/stylesheets/utilities/_break.scss"],"names":[],"mappings":"AA2BA,cAGE,6BAME,sDAAA,CACA,6DAAA,CACA,+DAAA,CACA,gEAAA,CACA,mDAAA,CACA,6DAAA,CACA,+DAAA,CACA,gEAAA,CAGA,mDAAA,CACA,gDAAA,CAGA,0BAAA,CACA,mCAAA,CAGA,iCAAA,CACA,kCAAA,CACA,mCAAA,CACA,mCAAA,CACA,kCAAA,CACA,iCAAA,CACA,+CAAA,CACA,6DAAA,CACA,gEAAA,CACA,4DAAA,CACA,4DAAA,CACA,6DAAA,CAGA,6CAAA,CAGA,+CAAA,CAGA,uDAAA,CACA,6DAAA,CACA,2DAAA,CAGA,iCAAA,CAGA,yDAAA,CACA,iEAAA,CAGA,mDAAA,CACA,mDAAA,CAGA,qDAAA,CACA,uDAAA,CAGA,8DAAA,CAKA,8DAAA,CAKA,0DAAA,CAvEA,iBCeF,CD6DE,kHAEE,YC3DJ,CDkFE,yDACE,4BChFJ,CD+EE,2DACE,4BC7EJ,CD4EE,gEACE,4BC1EJ,CDyEE,2DACE,4BCvEJ,CDsEE,yDACE,4BCpEJ,CDmEE,0DACE,4BCjEJ,CDgEE,gEACE,4BC9DJ,CD6DE,0DACE,4BC3DJ,CD0DE,2OACE,4BC/CJ,CDsDA,+FAGE,iCCpDF,CACF,CC/CE,2BACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCD2CN,CCrDE,4BACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCDkDN,CC5DE,8BACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCDyDN,CCnEE,mCACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCDgEN,CC1EE,8BACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCDuEN,CCjFE,4BACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCD8EN,CCxFE,kCACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCDqFN,CC/FE,4BACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCD4FN,CCtGE,4BACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCDmGN,CC7GE,6BACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCD0GN,CCpHE,mCACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCDiHN,CC3HE,4BACE,4BAAA,CACA,2CAAA,CAIE,8BAAA,CACA,qCD2HN,CClIE,8BACE,4BAAA,CACA,2CAAA,CAIE,8BAAA,CACA,qCDkIN,CCzIE,6BACE,yBAAA,CACA,2CAAA,CAIE,8BAAA,CACA,qCDyIN,CChJE,8BACE,4BAAA,CACA,2CAAA,CAIE,8BAAA,CACA,qCDgJN,CCvJE,mCACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCDoJN,CEzJE,4BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCFsJN,CEjKE,6BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCF8JN,CEzKE,+BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCFsKN,CEjLE,oCACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCF8KN,CEzLE,+BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCFsLN,CEjME,6BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCF8LN,CEzME,mCACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCFsMN,CEjNE,6BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCF8MN,CEzNE,6BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCFsNN,CEjOE,8BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCF8NN,CEzOE,oCACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCFsON,CEjPE,6BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAIE,+BAAA,CACA,sCFiPN,CEzPE,+BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAIE,+BAAA,CACA,sCFyPN,CEjQE,8BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAIE,+BAAA,CACA,sCFiQN,CEzQE,+BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAIE,+BAAA,CACA,sCFyQN,CEjRE,oCACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCF8QN,CEzRE,8BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCFsRN,CEjSE,6BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCAAA,CAKA,4BF0RN,CE1SE,kCACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCAAA,CAKA,4BFmSN,CEpRE,sEACE,4BFuRJ,CExRE,+DACE,4BF2RJ,CE5RE,iEACE,4BF+RJ,CEhSE,gEACE,4BFmSJ,CEpSE,iEACE,4BFuSJ,CE9RA,8BACE,mDAAA,CACA,4DAAA,CACA,0DAAA,CACA,oDAAA,CACA,2DAAA,CAGA,4BF+RF,CE5RE,yCACE,+BF8RJ,CE3RI,kDAEE,0CAAA,CACA,sCAAA,CAFA,mCF+RN,CG3MI,mCD1EA,+CACE,8CFwRJ,CErRI,qDACE,8CFuRN,CElRE,iEACE,mCFoRJ,CACF,CGtNI,sCDvDA,uCACE,oCFgRJ,CACF,CEvQA,8BACE,kDAAA,CACA,4DAAA,CACA,wDAAA,CACA,oDAAA,CACA,6DAAA,CAGA,4BFwQF,CErQE,yCACE,+BFuQJ,CEpQI,kDAEE,0CAAA,CACA,sCAAA,CAFA,mCFwQN,CEjQE,yCACE,6CFmQJ,CG5NI,0CDhCA,8CACE,gDF+PJ,CACF,CGjOI,0CDvBA,iFACE,6CF2PJ,CACF,CGzPI,sCDKA,uCACE,6CFuPJ,CACF","file":"palette.css"}
\ No newline at end of file
diff --git a/consultancy/index.html b/consultancy/index.html
index 633a8f374..8f9a921e7 100644
--- a/consultancy/index.html
+++ b/consultancy/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/consultancy/">
       
       <link rel="icon" href="../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -928,11 +944,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -960,14 +1100,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1020,14 +1160,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1158,14 +1298,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1181,14 +1321,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1241,14 +1381,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1518,7 +1658,7 @@ <h1 id="training-consultancy-deployment">Training, Consultancy &amp; Deployment<
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1581,12 +1721,16 @@ <h1 id="training-consultancy-deployment">Training, Consultancy &amp; Deployment<
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/data_engineering/nosql/cassandra/intro/index.html b/data_engineering/nosql/cassandra/intro/index.html
index f93dfed2a..574dffa9c 100644
--- a/data_engineering/nosql/cassandra/intro/index.html
+++ b/data_engineering/nosql/cassandra/intro/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/data_engineering/nosql/cassandra/intro/">
       
       <link rel="icon" href="../../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1008,14 +1148,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1148,14 +1288,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" checked>
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1173,14 +1313,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" checked>
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1273,14 +1413,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1498,7 +1638,7 @@
                 
                 
                 <h1 id="apache-cassandra">Apache Cassandra<a class="headerlink" href="#apache-cassandra" title="Permanent link">&para;</a></h1>
-<div class="tabbed-set" data-tabs="1:1"><input checked="checked" id="__tabbed_1_1" name="__tabbed_1" type="radio" /><label for="__tabbed_1_1">Tools You Will Learn To Use</label><div class="tabbed-content">
+<div class="tabbed-set" data-tabs="1:1"><input checked="checked" id="apache-cassandra-tools-you-will-learn-to-use" name="__tabbed_1" type="radio" /><label for="apache-cassandra-tools-you-will-learn-to-use">Tools You Will Learn To Use</label><div class="tabbed-content">
 <ul>
 <li>Apache Cassandra</li>
 </ul>
@@ -1585,7 +1725,7 @@ <h2 id="reasons-for-using-apache-cassandra">Reasons for Using Apache Cassandra<a
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1648,12 +1788,16 @@ <h2 id="reasons-for-using-apache-cassandra">Reasons for Using Apache Cassandra<a
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/data_engineering/nosql/cassandra/setting_up_cluster/index.html b/data_engineering/nosql/cassandra/setting_up_cluster/index.html
index 069cec2b7..6700d7470 100644
--- a/data_engineering/nosql/cassandra/setting_up_cluster/index.html
+++ b/data_engineering/nosql/cassandra/setting_up_cluster/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/data_engineering/nosql/cassandra/setting_up_cluster/">
       
       <link rel="icon" href="../../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1008,14 +1148,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1148,14 +1288,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" checked>
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1173,14 +1313,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" checked>
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1403,14 +1543,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -2053,7 +2193,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -2116,12 +2256,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/index.html b/deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/index.html
index 8abf1a159..bf5ffbcd4 100644
--- a/deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/index.html
+++ b/deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -1022,11 +1038,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1054,14 +1194,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1114,14 +1254,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1252,14 +1392,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1275,14 +1415,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1335,14 +1475,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1829,7 +1969,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1892,12 +2032,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/boosting_models_pytorch/forwardpropagation_backpropagation_gradientdescent/index.html b/deep_learning/boosting_models_pytorch/forwardpropagation_backpropagation_gradientdescent/index.html
index a4668096f..63fc4e5c9 100644
--- a/deep_learning/boosting_models_pytorch/forwardpropagation_backpropagation_gradientdescent/index.html
+++ b/deep_learning/boosting_models_pytorch/forwardpropagation_backpropagation_gradientdescent/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/boosting_models_pytorch/forwardpropagation_backpropagation_gradientdescent/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -981,11 +997,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1013,14 +1153,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1073,14 +1213,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1211,14 +1351,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1234,14 +1374,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1294,14 +1434,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1673,7 +1813,7 @@ <h2 id="forward-propagation-backward-propagation-and-gradient-descent">Forward P
         <span class="c1"># Derivative of binary cross entropy cost w.r.t. final output y4</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">dC_dy4</span> <span class="o">=</span> <span class="n">y4</span> <span class="o">-</span> <span class="n">l</span>
 
-        <span class="sd">&#39;&#39;&#39;</span>
+<span class="w">        </span><span class="sd">&#39;&#39;&#39;</span>
 <span class="sd">        Gradients for w2: partial derivative of cost w.r.t. w2</span>
 <span class="sd">        dC/dw2</span>
 <span class="sd">        &#39;&#39;&#39;</span>
@@ -1686,7 +1826,7 @@ <h2 id="forward-propagation-backward-propagation-and-gradient-descent">Forward P
         <span class="c1"># This is our gradients for w1: dC_dy4 dy4_dy3 dy3_dw2</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">dC_dw2</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">matmul</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">t</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dy3_dw2</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">y4_delta</span><span class="p">)</span>
 
-        <span class="sd">&#39;&#39;&#39;</span>
+<span class="w">        </span><span class="sd">&#39;&#39;&#39;</span>
 <span class="sd">        Gradients for w1: partial derivative of cost w.r.t w1</span>
 <span class="sd">        dC/dw1</span>
 <span class="sd">        &#39;&#39;&#39;</span>
@@ -1830,7 +1970,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1893,12 +2033,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/boosting_models_pytorch/lr_scheduling/index.html b/deep_learning/boosting_models_pytorch/lr_scheduling/index.html
index 90ccadb2b..dc06daa33 100644
--- a/deep_learning/boosting_models_pytorch/lr_scheduling/index.html
+++ b/deep_learning/boosting_models_pytorch/lr_scheduling/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/boosting_models_pytorch/lr_scheduling/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -1083,11 +1099,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1115,14 +1255,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1175,14 +1315,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1313,14 +1453,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1336,14 +1476,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1396,14 +1536,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -2934,7 +3074,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -2997,12 +3137,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/boosting_models_pytorch/optimizers/index.html b/deep_learning/boosting_models_pytorch/optimizers/index.html
index d8996d2ec..e2e5c1861 100644
--- a/deep_learning/boosting_models_pytorch/optimizers/index.html
+++ b/deep_learning/boosting_models_pytorch/optimizers/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/boosting_models_pytorch/optimizers/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -1106,11 +1122,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1138,14 +1278,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1198,14 +1338,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1336,14 +1476,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1359,14 +1499,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1419,14 +1559,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -3385,7 +3525,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -3448,12 +3588,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/index.html b/deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/index.html
index 8c7afb59f..6e207b27c 100644
--- a/deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/index.html
+++ b/deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -1165,11 +1181,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1197,14 +1337,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1257,14 +1397,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1395,14 +1535,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1418,14 +1558,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1478,14 +1618,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -2977,30 +3117,30 @@ <h2 id="initialization-performance">Initialization Performance<a class="headerli
 <table>
 <thead>
 <tr>
-<th align="center">Initialization: Activation</th>
-<th align="center">Test Accuracy</th>
+<th style="text-align: center;">Initialization: Activation</th>
+<th style="text-align: center;">Test Accuracy</th>
 </tr>
 </thead>
 <tbody>
 <tr>
-<td align="center">Normal: Tanh</td>
-<td align="center">92.55</td>
+<td style="text-align: center;">Normal: Tanh</td>
+<td style="text-align: center;">92.55</td>
 </tr>
 <tr>
-<td align="center">Lecun: Tanh</td>
-<td align="center">97.7</td>
+<td style="text-align: center;">Lecun: Tanh</td>
+<td style="text-align: center;">97.7</td>
 </tr>
 <tr>
-<td align="center">Xavier: Tanh</td>
-<td align="center">97.49</td>
+<td style="text-align: center;">Xavier: Tanh</td>
+<td style="text-align: center;">97.49</td>
 </tr>
 <tr>
-<td align="center">Xavier: ReLU</td>
-<td align="center">97.48</td>
+<td style="text-align: center;">Xavier: ReLU</td>
+<td style="text-align: center;">97.48</td>
 </tr>
 <tr>
-<td align="center">He: ReLU</td>
-<td align="center">97.39</td>
+<td style="text-align: center;">He: ReLU</td>
+<td style="text-align: center;">97.39</td>
 </tr>
 </tbody>
 </table>
@@ -3118,7 +3258,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -3181,12 +3321,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/course_progression/index.html b/deep_learning/course_progression/index.html
index 4903e1811..27b8591e7 100644
--- a/deep_learning/course_progression/index.html
+++ b/deep_learning/course_progression/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/course_progression/">
       
       <link rel="icon" href="../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -986,11 +1002,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1018,14 +1158,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1078,14 +1218,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1216,14 +1356,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1239,14 +1379,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1299,14 +1439,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1552,7 +1692,7 @@
                 
                 
                 <h1 id="course-progression">Course Progression<a class="headerlink" href="#course-progression" title="Permanent link">&para;</a></h1>
-<p>If you would like a smooth transition in learning deep learning concepts, you need to follow the materials in a sequential order. Some sections are still pending as I am working on them, and they will have the <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> icon beside them.</p>
+<p>If you would like a smooth transition in learning deep learning concepts, you need to follow the materials in a sequential order. Some sections are still pending as I am working on them, and they will have the <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> icon beside them.</p>
 <h2 id="1-practical-deep-learning-with-pytorch">1. Practical Deep Learning with PyTorch<a class="headerlink" href="#1-practical-deep-learning-with-pytorch" title="Permanent link">&para;</a></h2>
 <ul class="task-list">
 <li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> Matrices</li>
@@ -1565,10 +1705,10 @@ <h2 id="1-practical-deep-learning-with-pytorch">1. Practical Deep Learning with
 <li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> Long Short Term Memory Neural Networks (LSTM)</li>
 <li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> Autoencoders (AE)</li>
 <li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> Fully-connected Overcomplete Autoencoder (AE)</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Variational Autoencoders (VAE)</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Adversarial Autoencoders (AAE)</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Generative Adversarial Networks (GAN)</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Transformers</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Variational Autoencoders (VAE)</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Adversarial Autoencoders (AAE)</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Generative Adversarial Networks (GAN)</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Transformers</li>
 </ul>
 <h2 id="2-improving-deep-learning-models-with-pytorch">2. Improving Deep Learning Models with PyTorch<a class="headerlink" href="#2-improving-deep-learning-models-with-pytorch" title="Permanent link">&para;</a></h2>
 <ul class="task-list">
@@ -1578,39 +1718,39 @@ <h2 id="2-improving-deep-learning-models-with-pytorch">2. Improving Deep Learnin
 <li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> Optimizers</li>
 <li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> Advanced Learning Rate Optimization</li>
 <li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> Weight Initializations and Activation Functions</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Overfitting Prevention</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Loss, Accuracy and Weight Visualizations</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Data Preprocessing for Images and Videos</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Data Preprocessing for Time Series</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Overfitting Prevention</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Loss, Accuracy and Weight Visualizations</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Data Preprocessing for Images and Videos</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Data Preprocessing for Time Series</li>
 </ul>
 <h2 id="3-deep-reinforcement-learning-with-pytorch">3. Deep Reinforcement Learning with PyTorch<a class="headerlink" href="#3-deep-reinforcement-learning-with-pytorch" title="Permanent link">&para;</a></h2>
 <ul class="task-list">
 <li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> Supervised Learning to Reinforcement Learning</li>
 <li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> Markov Decision Processes and Bellman Equations</li>
 <li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> Dynamic Programming</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Monte Carlo Approach</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Temporal-Difference</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Policy Gradient: REINFORCE</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Policy Gradient: Actor-Critic</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Policy Gradient: A2C/A3C</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Policy Gradient: ACKTR</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Policy Gradient: PPO</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Policy Gradient: DPG</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Policy Gradient: DDPG (DQN &amp; DPG)</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Monte Carlo Approach</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Temporal-Difference</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Policy Gradient: REINFORCE</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Policy Gradient: Actor-Critic</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Policy Gradient: A2C/A3C</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Policy Gradient: ACKTR</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Policy Gradient: PPO</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Policy Gradient: DPG</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Policy Gradient: DDPG (DQN &amp; DPG)</li>
 </ul>
 <h2 id="4-from-scratch-with-python-and-pytorch">4. From Scratch with Python and PyTorch<a class="headerlink" href="#4-from-scratch-with-python-and-pytorch" title="Permanent link">&para;</a></h2>
 <ul class="task-list">
 <li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> From Scratch Logistic Regression Classification</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> From Scratch FNN Classification</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> From Scratch CNN Classification</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> From Scratch RNN Classification</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> From Scratch LSTM Classification</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> From Scratch AE</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> From Scratch FNN Classification</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> From Scratch CNN Classification</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> From Scratch RNN Classification</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> From Scratch LSTM Classification</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> From Scratch AE</li>
 </ul>
 <h2 id="5-large-language-models-with-pytorch">5. Large Language Models with PyTorch<a class="headerlink" href="#5-large-language-models-with-pytorch" title="Permanent link">&para;</a></h2>
 <ul class="task-list">
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Introduction</li>
-<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/23f3.svg" title=":hourglass_flowing_sand:" /> What is Temperature</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> Introduction</li>
+<li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> <img alt="⏳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/23f3.svg" title=":hourglass_flowing_sand:" /> What is Temperature</li>
 </ul>
                 
                   
@@ -1677,7 +1817,7 @@ <h2 id="5-large-language-models-with-pytorch">5. Large Language Models with PyTo
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1740,12 +1880,16 @@ <h2 id="5-large-language-models-with-pytorch">5. Large Language Models with PyTo
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/index.html b/deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/index.html
index 1c793fe5c..7a1abfbd7 100644
--- a/deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/index.html
+++ b/deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -1052,11 +1068,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1084,14 +1224,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1144,14 +1284,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1282,14 +1422,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1305,14 +1445,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1365,14 +1505,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -2030,7 +2170,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -2093,12 +2233,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/index.html b/deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/index.html
index 030419796..26c7eb16e 100644
--- a/deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/index.html
+++ b/deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -1119,11 +1135,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1151,14 +1291,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1211,14 +1351,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1349,14 +1489,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1372,14 +1512,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1432,14 +1572,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -2349,7 +2489,7 @@ <h3 id="custom-frozen-lake-non-deterministic-policy-environment">Custom Frozen L
 
 
 <span class="k">class</span> <span class="nc">FrozenLakeEnv</span><span class="p">(</span><span class="n">discrete</span><span class="o">.</span><span class="n">DiscreteEnv</span><span class="p">):</span>
-    <span class="sd">&quot;&quot;&quot;</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Winter is here. You and your friends were tossing around a frisbee at the park</span>
 <span class="sd">    when you made a wild throw that left the frisbee out in the middle of the lake.</span>
 <span class="sd">    The water is mostly frozen, but there are a few holes where the ice has melted.</span>
@@ -2533,7 +2673,7 @@ <h3 id="policy-evaluation-function-comprising-state-value-function">Policy Evalu
 <div class="highlight"><pre><span></span><code><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
 
 <span class="k">def</span> <span class="nf">policy_evaluation</span><span class="p">(</span><span class="n">env</span><span class="p">,</span> <span class="n">policy</span><span class="p">,</span> <span class="n">gamma</span><span class="o">=</span><span class="mf">1.</span><span class="p">,</span> <span class="n">theta</span><span class="o">=</span><span class="mf">1e-8</span><span class="p">):</span>
-    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Policy evaluation function. Loop until state values stable, delta &lt; theta.</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Policy evaluation function. Loop until state values stable, delta &lt; theta.</span>
 
 <span class="sd">    Returns V comprising values of states under given policy.</span>
 
@@ -2609,7 +2749,7 @@ <h3 id="action-value-q-value-function-from-state-value-function">Action-value (Q
 <li><strong>Aim: getting q-values (action-values)</strong></li>
 </ul>
 <div class="highlight"><pre><span></span><code><span class="k">def</span> <span class="nf">q_value</span><span class="p">(</span><span class="n">env</span><span class="p">,</span> <span class="n">V</span><span class="p">,</span> <span class="n">s</span><span class="p">,</span> <span class="n">gamma</span><span class="o">=</span><span class="mi">1</span><span class="p">):</span>
-    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Q-value (action-value) function from state-value function</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Q-value (action-value) function from state-value function</span>
 
 <span class="sd">    Returns Q values, values of actions.</span>
 
@@ -2659,7 +2799,7 @@ <h3 id="policy-improvement-function">Policy Improvement Function<a class="header
 <li>Aim: get improved policy</li>
 </ul>
 <div class="highlight"><pre><span></span><code><span class="k">def</span> <span class="nf">policy_improvement</span><span class="p">(</span><span class="n">env</span><span class="p">,</span> <span class="n">V</span><span class="p">,</span> <span class="n">gamma</span><span class="o">=</span><span class="mf">1.</span><span class="p">):</span>
-    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Function to improve the policy by utilizing state values and action (q) values.</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Function to improve the policy by utilizing state values and action (q) values.</span>
 
 <span class="sd">    Args:</span>
 <span class="sd">        env (gym.env): OpenAI environment class instantiated and assigned to an objects</span>
@@ -2865,7 +3005,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -2928,12 +3068,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/deep_reinforcement_learning_pytorch/rl_notes/index.html b/deep_learning/deep_reinforcement_learning_pytorch/rl_notes/index.html
index 2a4d3d765..78545a8a0 100644
--- a/deep_learning/deep_reinforcement_learning_pytorch/rl_notes/index.html
+++ b/deep_learning/deep_reinforcement_learning_pytorch/rl_notes/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/deep_reinforcement_learning_pytorch/rl_notes/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -181,7 +181,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -256,6 +256,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -350,7 +366,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -909,11 +925,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -941,14 +1081,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1001,14 +1141,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1139,14 +1279,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1162,14 +1302,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1222,14 +1362,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1447,7 +1587,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1510,12 +1650,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/deep_reinforcement_learning_pytorch/supervised_to_rl/index.html b/deep_learning/deep_reinforcement_learning_pytorch/supervised_to_rl/index.html
index 142344cb8..eeb97924f 100644
--- a/deep_learning/deep_reinforcement_learning_pytorch/supervised_to_rl/index.html
+++ b/deep_learning/deep_reinforcement_learning_pytorch/supervised_to_rl/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/deep_reinforcement_learning_pytorch/supervised_to_rl/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -974,11 +990,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1006,14 +1146,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1066,14 +1206,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1204,14 +1344,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1227,14 +1367,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1287,14 +1427,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1690,7 +1830,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1753,12 +1893,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/fromscratch/fromscratch_cnn/index.html b/deep_learning/fromscratch/fromscratch_cnn/index.html
index 2241d0399..053475a45 100644
--- a/deep_learning/fromscratch/fromscratch_cnn/index.html
+++ b/deep_learning/fromscratch/fromscratch_cnn/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/fromscratch/fromscratch_cnn/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -355,7 +371,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -914,11 +930,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -946,14 +1086,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1006,14 +1146,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1144,14 +1284,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1167,14 +1307,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1227,14 +1367,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1638,7 +1778,7 @@ <h3 id="convolutional-layer">Convolutional Layer<a class="headerlink" href="#con
         <span class="bp">self</span><span class="o">.</span><span class="n">kernels_theta</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">num_kernels</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">kernel_shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">kernel_shape</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
 
     <span class="k">def</span> <span class="nf">slider</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inp</span><span class="p">):</span>
-        <span class="sd">&#39;&#39;&#39;</span>
+<span class="w">        </span><span class="sd">&#39;&#39;&#39;</span>
 <span class="sd">        Sliding generator that yields square areas of shape</span>
 <span class="sd">        (kernel_shape, kernel_shape) sliding across our input. </span>
 <span class="sd">        This assumes valid padding (no padding) and step size 1.</span>
@@ -1650,7 +1790,7 @@ <h3 id="convolutional-layer">Convolutional Layer<a class="headerlink" href="#con
                 <span class="k">yield</span> <span class="n">single_slide_area</span><span class="p">,</span> <span class="n">h_idx</span><span class="p">,</span><span class="n">w_idx</span>
 
     <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inp</span><span class="p">):</span>
-        <span class="sd">&#39;&#39;&#39;</span>
+<span class="w">        </span><span class="sd">&#39;&#39;&#39;</span>
 <span class="sd">        Slides kernel across image doing an element-wise MM then summing.</span>
 <span class="sd">        Results in forward pass of convolutional layer of shape</span>
 <span class="sd">        (output shape, output shape, number of kernels).</span>
@@ -1723,7 +1863,7 @@ <h3 id="max-pooling-layer">Max Pooling Layer<a class="headerlink" href="#max-poo
         <span class="bp">self</span><span class="o">.</span><span class="n">k</span> <span class="o">=</span> <span class="n">pooling_kernel_shape</span>
 
     <span class="k">def</span> <span class="nf">slider</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inp</span><span class="p">):</span>
-        <span class="sd">&#39;&#39;&#39;</span>
+<span class="w">        </span><span class="sd">&#39;&#39;&#39;</span>
 <span class="sd">        Sliding generator that yields areas for max pooling.</span>
 <span class="sd">        &#39;&#39;&#39;</span>
         <span class="n">h</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">inp</span><span class="o">.</span><span class="n">shape</span>
@@ -1736,7 +1876,7 @@ <h3 id="max-pooling-layer">Max Pooling Layer<a class="headerlink" href="#max-poo
                 <span class="k">yield</span> <span class="n">single_slide_area</span><span class="p">,</span> <span class="n">h_idx</span><span class="p">,</span> <span class="n">w_idx</span>
 
     <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inp</span><span class="p">):</span>
-        <span class="sd">&#39;&#39;&#39;</span>
+<span class="w">        </span><span class="sd">&#39;&#39;&#39;</span>
 <span class="sd">        Performs a forward pass of the maxpool layer using the given input.</span>
 <span class="sd">        Returns a 3d numpy array with dimensions (h / 2, w / 2, num_filters).</span>
 <span class="sd">        - input is a 3d numpy array with dimensions (h, w, num_filters)</span>
@@ -1788,7 +1928,7 @@ <h3 id="affine-and-softargmax-layer">Affine and Soft(arg)max Layer<a class="head
         <span class="bp">self</span><span class="o">.</span><span class="n">b</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">normal_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">b</span><span class="p">,</span> <span class="n">mean</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">std</span><span class="o">=</span><span class="mi">1</span><span class="o">/</span><span class="bp">self</span><span class="o">.</span><span class="n">affine_weight_shape</span><span class="p">[</span><span class="mi">3</span><span class="p">])</span>
 
     <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inp</span><span class="p">):</span>
-        <span class="sd">&#39;&#39;&#39;</span>
+<span class="w">        </span><span class="sd">&#39;&#39;&#39;</span>
 <span class="sd">        Performs Linear (Affine) Function &amp; Soft(arg)max Function</span>
 <span class="sd">        that returns our vector (1D) of probabilities.</span>
 <span class="sd">        &#39;&#39;&#39;</span>
@@ -1813,7 +1953,7 @@ <h3 id="affine-and-softargmax-layer">Affine and Soft(arg)max Layer<a class="head
 <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Affine &amp; Soft(arg)max (f) shape: </span><span class="se">\t</span><span class="s1"> </span><span class="si">{</span><span class="nb">list</span><span class="p">(</span><span class="n">output</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>
 <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;=&#39;</span><span class="o">*</span><span class="mi">50</span><span class="p">)</span>
 
-<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Probas: </span><span class="si">{</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">output</span><span class="o">.</span><span class="n">numpy</span><span class="p">())</span><span class="o">.</span><span class="n">to_string</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">header</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>
+<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Probas: </span><span class="si">{</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">output</span><span class="o">.</span><span class="n">numpy</span><span class="p">())</span><span class="o">.</span><span class="n">to_string</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span><span class="w"> </span><span class="n">header</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>==================================================
 Input shape:     [12, 12, 8]
@@ -1947,7 +2087,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -2010,12 +2150,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/fromscratch/fromscratch_logistic_regression/index.html b/deep_learning/fromscratch/fromscratch_logistic_regression/index.html
index 38a4860f3..4f3f8590b 100644
--- a/deep_learning/fromscratch/fromscratch_logistic_regression/index.html
+++ b/deep_learning/fromscratch/fromscratch_logistic_regression/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/fromscratch/fromscratch_logistic_regression/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -1062,11 +1078,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1094,14 +1234,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1154,14 +1294,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1292,14 +1432,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1315,14 +1455,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1375,14 +1515,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1864,7 +2004,7 @@ <h2 id="training-from-scratch">Training from Scratch<a class="headerlink" href="
     <span class="n">acc</span> <span class="o">=</span> <span class="mi">100</span> <span class="o">*</span> <span class="n">correct</span> <span class="o">/</span> <span class="n">total</span> 
 
     <span class="c1"># Print accuracy and cost</span>
-    <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Epoch: </span><span class="si">{</span><span class="n">epoch</span><span class="si">}</span><span class="s1"> | Accuracy: </span><span class="si">{</span><span class="n">acc</span><span class="o">.</span><span class="n">item</span><span class="p">()</span> <span class="si">:</span><span class="s1">.4f</span><span class="si">}</span><span class="s1"> | Cost: </span><span class="si">{</span><span class="n">l</span><span class="o">.</span><span class="n">item</span><span class="p">()</span> <span class="si">:</span><span class="s1">.4f</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>
+    <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Epoch: </span><span class="si">{</span><span class="n">epoch</span><span class="si">}</span><span class="s1"> | Accuracy: </span><span class="si">{</span><span class="n">acc</span><span class="o">.</span><span class="n">item</span><span class="p">()</span><span class="w"> </span><span class="si">:</span><span class="s1">.4f</span><span class="si">}</span><span class="s1"> | Cost: </span><span class="si">{</span><span class="n">l</span><span class="o">.</span><span class="n">item</span><span class="p">()</span><span class="w"> </span><span class="si">:</span><span class="s1">.4f</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>
 
 <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Weights </span><span class="se">\n</span><span class="s1"> </span><span class="si">{</span><span class="n">weights</span><span class="o">.</span><span class="n">data</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>
 <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Bias </span><span class="se">\n</span><span class="s1"> </span><span class="si">{</span><span class="n">bias</span><span class="o">.</span><span class="n">data</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>
@@ -1917,7 +2057,7 @@ <h2 id="inference">Inference<a class="headerlink" href="#inference" title="Perma
 <span class="n">acc</span> <span class="o">=</span> <span class="mi">100</span> <span class="o">*</span> <span class="n">correct</span> <span class="o">/</span> <span class="n">total</span> 
 
 <span class="c1"># Print accuracy</span>
-<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Validation Accuracy: </span><span class="si">{</span><span class="n">acc</span><span class="o">.</span><span class="n">item</span><span class="p">()</span> <span class="si">:</span><span class="s1">.4f</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>
+<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Validation Accuracy: </span><span class="si">{</span><span class="n">acc</span><span class="o">.</span><span class="n">item</span><span class="p">()</span><span class="w"> </span><span class="si">:</span><span class="s1">.4f</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Validation Accuracy: 100.0000
 </code></pre></div>
@@ -1986,7 +2126,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -2049,12 +2189,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/intro/index.html b/deep_learning/intro/index.html
index eabf81ac8..a67c4b722 100644
--- a/deep_learning/intro/index.html
+++ b/deep_learning/intro/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/intro/">
       
       <link rel="icon" href="../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -958,11 +974,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -990,14 +1130,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1050,14 +1190,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1188,14 +1328,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1211,14 +1351,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1271,14 +1411,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1497,7 +1637,7 @@
                 
                 <h1 id="deep-learning-and-deep-reinforcement-learning-theory-and-programming-tutorials">Deep Learning and Deep Reinforcement Learning Theory and Programming Tutorials<a class="headerlink" href="#deep-learning-and-deep-reinforcement-learning-theory-and-programming-tutorials" title="Permanent link">&para;</a></h1>
 <p>We'll be covering both CPU and GPU implementations of deep learning and deep reinforcement learning algorithms</p>
-<div class="tabbed-set" data-tabs="1:1"><input checked="checked" id="__tabbed_1_1" name="__tabbed_1" type="radio" /><label for="__tabbed_1_1">Packages and Languages you will Learn to Use</label><div class="tabbed-content">
+<div class="tabbed-set" data-tabs="1:1"><input checked="checked" id="deep-learning-and-deep-reinforcement-learning-theory-and-programming-tutorials-packages-and-languages-you-will-learn-to-use" name="__tabbed_1" type="radio" /><label for="deep-learning-and-deep-reinforcement-learning-theory-and-programming-tutorials-packages-and-languages-you-will-learn-to-use">Packages and Languages you will Learn to Use</label><div class="tabbed-content">
 <ul>
 <li>Python</li>
 <li>PyTorch</li>
@@ -1598,7 +1738,7 @@ <h2 id="paths">Paths<a class="headerlink" href="#paths" title="Permanent link">&
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1661,12 +1801,16 @@ <h2 id="paths">Paths<a class="headerlink" href="#paths" title="Permanent link">&
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/practical_pytorch/pytorch_autoencoder/index.html b/deep_learning/practical_pytorch/pytorch_autoencoder/index.html
index 1aa7c9f5a..494d6a750 100644
--- a/deep_learning/practical_pytorch/pytorch_autoencoder/index.html
+++ b/deep_learning/practical_pytorch/pytorch_autoencoder/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_autoencoder/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -994,11 +1010,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1026,14 +1166,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1086,14 +1226,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1224,14 +1364,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1247,14 +1387,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1307,14 +1447,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1705,7 +1845,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1768,12 +1908,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/index.html b/deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/index.html
index e6f9fb91a..ac50b374e 100644
--- a/deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/index.html
+++ b/deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -1279,11 +1295,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1311,14 +1451,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1371,14 +1511,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1509,14 +1649,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1532,14 +1672,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1592,14 +1732,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -2284,7 +2424,7 @@ <h4 id="step-1-loading-mnist-train-dataset">Step 1: Loading MNIST Train Dataset<
 <div class="highlight"><pre><span></span><code><span class="nb">print</span><span class="p">(</span><span class="n">train_dataset</span><span class="o">.</span><span class="n">train_data</span><span class="o">.</span><span class="n">size</span><span class="p">())</span>
 </code></pre></div>
 </div>
-<div class="highlight"><pre><span></span><code>torch.Size<span class="o">([</span><span class="m">60000</span>, <span class="m">28</span>, <span class="m">28</span><span class="o">])</span>
+<div class="highlight"><pre><span></span><code>torch.Size<span class="o">([</span><span class="m">60000</span>,<span class="w"> </span><span class="m">28</span>,<span class="w"> </span><span class="m">28</span><span class="o">])</span>
 </code></pre></div>
 <div class="admonition note">
 <p class="admonition-title">Size of our training dataset labels</p>
@@ -2298,7 +2438,7 @@ <h4 id="step-1-loading-mnist-train-dataset">Step 1: Loading MNIST Train Dataset<
 </div>
 <div class="highlight"><pre><span></span><code><span class="nb">print</span><span class="p">(</span><span class="n">test_dataset</span><span class="o">.</span><span class="n">test_data</span><span class="o">.</span><span class="n">size</span><span class="p">())</span>
 </code></pre></div>
-<div class="highlight"><pre><span></span><code>torch.Size<span class="o">([</span><span class="m">10000</span>, <span class="m">28</span>, <span class="m">28</span><span class="o">])</span>
+<div class="highlight"><pre><span></span><code>torch.Size<span class="o">([</span><span class="m">10000</span>,<span class="w"> </span><span class="m">28</span>,<span class="w"> </span><span class="m">28</span><span class="o">])</span>
 </code></pre></div>
 <div class="admonition note">
 <p class="admonition-title">Size of our testing dataset labels</p>
@@ -2474,13 +2614,13 @@ <h5 id="parameters-in-depth">Parameters In-Depth<a class="headerlink" href="#par
 <span class="nb">print</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">model</span><span class="o">.</span><span class="n">parameters</span><span class="p">())[</span><span class="mi">5</span><span class="p">]</span><span class="o">.</span><span class="n">size</span><span class="p">())</span>
 </code></pre></div>
 </div>
-<div class="highlight"><pre><span></span><code>&lt;generator object Module.parameters at 0x7f9864363c50&gt;
+<div class="highlight"><pre><span></span><code>&lt;generator<span class="w"> </span>object<span class="w"> </span>Module.parameters<span class="w"> </span>at<span class="w"> </span>0x7f9864363c50&gt;
 <span class="m">6</span>
-torch.Size<span class="o">([</span><span class="m">16</span>, <span class="m">1</span>, <span class="m">5</span>, <span class="m">5</span><span class="o">])</span>
+torch.Size<span class="o">([</span><span class="m">16</span>,<span class="w"> </span><span class="m">1</span>,<span class="w"> </span><span class="m">5</span>,<span class="w"> </span><span class="m">5</span><span class="o">])</span>
 torch.Size<span class="o">([</span><span class="m">16</span><span class="o">])</span>
-torch.Size<span class="o">([</span><span class="m">32</span>, <span class="m">16</span>, <span class="m">5</span>, <span class="m">5</span><span class="o">])</span>
+torch.Size<span class="o">([</span><span class="m">32</span>,<span class="w"> </span><span class="m">16</span>,<span class="w"> </span><span class="m">5</span>,<span class="w"> </span><span class="m">5</span><span class="o">])</span>
 torch.Size<span class="o">([</span><span class="m">32</span><span class="o">])</span>
-torch.Size<span class="o">([</span><span class="m">10</span>, <span class="m">1568</span><span class="o">])</span>
+torch.Size<span class="o">([</span><span class="m">10</span>,<span class="w"> </span><span class="m">1568</span><span class="o">])</span>
 torch.Size<span class="o">([</span><span class="m">10</span><span class="o">])</span>
 </code></pre></div>
 <h4 id="step-7-train-model">Step 7: Train Model<a class="headerlink" href="#step-7-train-model" title="Permanent link">&para;</a></h4>
@@ -2742,12 +2882,12 @@ <h4 id="steps_1">Steps<a class="headerlink" href="#steps_1" title="Permanent lin
             <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Iteration: </span><span class="si">{}</span><span class="s1">. Loss: </span><span class="si">{}</span><span class="s1">. Accuracy: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">iter</span><span class="p">,</span> <span class="n">loss</span><span class="o">.</span><span class="n">item</span><span class="p">(),</span> <span class="n">accuracy</span><span class="p">))</span>
 </code></pre></div>
 </div>
-<div class="highlight"><pre><span></span><code>Iteration: <span class="m">500</span>. Loss: <span class="m">0</span>.6850348711013794. Accuracy: <span class="m">85</span>
-Iteration: <span class="m">1000</span>. Loss: <span class="m">0</span>.36549052596092224. Accuracy: <span class="m">88</span>
-Iteration: <span class="m">1500</span>. Loss: <span class="m">0</span>.31540098786354065. Accuracy: <span class="m">89</span>
-Iteration: <span class="m">2000</span>. Loss: <span class="m">0</span>.3522164225578308. Accuracy: <span class="m">90</span>
-Iteration: <span class="m">2500</span>. Loss: <span class="m">0</span>.2680729925632477. Accuracy: <span class="m">91</span>
-Iteration: <span class="m">3000</span>. Loss: <span class="m">0</span>.26440390944480896. Accuracy: <span class="m">92</span>
+<div class="highlight"><pre><span></span><code>Iteration:<span class="w"> </span><span class="m">500</span>.<span class="w"> </span>Loss:<span class="w"> </span><span class="m">0</span>.6850348711013794.<span class="w"> </span>Accuracy:<span class="w"> </span><span class="m">85</span>
+Iteration:<span class="w"> </span><span class="m">1000</span>.<span class="w"> </span>Loss:<span class="w"> </span><span class="m">0</span>.36549052596092224.<span class="w"> </span>Accuracy:<span class="w"> </span><span class="m">88</span>
+Iteration:<span class="w"> </span><span class="m">1500</span>.<span class="w"> </span>Loss:<span class="w"> </span><span class="m">0</span>.31540098786354065.<span class="w"> </span>Accuracy:<span class="w"> </span><span class="m">89</span>
+Iteration:<span class="w"> </span><span class="m">2000</span>.<span class="w"> </span>Loss:<span class="w"> </span><span class="m">0</span>.3522164225578308.<span class="w"> </span>Accuracy:<span class="w"> </span><span class="m">90</span>
+Iteration:<span class="w"> </span><span class="m">2500</span>.<span class="w"> </span>Loss:<span class="w"> </span><span class="m">0</span>.2680729925632477.<span class="w"> </span>Accuracy:<span class="w"> </span><span class="m">91</span>
+Iteration:<span class="w"> </span><span class="m">3000</span>.<span class="w"> </span>Loss:<span class="w"> </span><span class="m">0</span>.26440390944480896.<span class="w"> </span>Accuracy:<span class="w"> </span><span class="m">92</span>
 </code></pre></div>
 <div class="admonition note">
 <p class="admonition-title">Comparison of accuracies</p>
@@ -2933,12 +3073,12 @@ <h4 id="steps_2">Steps<a class="headerlink" href="#steps_2" title="Permanent lin
             <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Iteration: </span><span class="si">{}</span><span class="s1">. Loss: </span><span class="si">{}</span><span class="s1">. Accuracy: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">iter</span><span class="p">,</span> <span class="n">loss</span><span class="o">.</span><span class="n">item</span><span class="p">(),</span> <span class="n">accuracy</span><span class="p">))</span>
 </code></pre></div>
 </div>
-<div class="highlight"><pre><span></span><code>Iteration: <span class="m">500</span>. Loss: <span class="m">0</span>.5153220295906067. Accuracy: <span class="m">88</span>
-Iteration: <span class="m">1000</span>. Loss: <span class="m">0</span>.28784745931625366. Accuracy: <span class="m">92</span>
-Iteration: <span class="m">1500</span>. Loss: <span class="m">0</span>.4086027443408966. Accuracy: <span class="m">94</span>
-Iteration: <span class="m">2000</span>. Loss: <span class="m">0</span>.09390712529420853. Accuracy: <span class="m">95</span>
-Iteration: <span class="m">2500</span>. Loss: <span class="m">0</span>.07138358801603317. Accuracy: <span class="m">95</span>
-Iteration: <span class="m">3000</span>. Loss: <span class="m">0</span>.05396252125501633. Accuracy: <span class="m">96</span>
+<div class="highlight"><pre><span></span><code>Iteration:<span class="w"> </span><span class="m">500</span>.<span class="w"> </span>Loss:<span class="w"> </span><span class="m">0</span>.5153220295906067.<span class="w"> </span>Accuracy:<span class="w"> </span><span class="m">88</span>
+Iteration:<span class="w"> </span><span class="m">1000</span>.<span class="w"> </span>Loss:<span class="w"> </span><span class="m">0</span>.28784745931625366.<span class="w"> </span>Accuracy:<span class="w"> </span><span class="m">92</span>
+Iteration:<span class="w"> </span><span class="m">1500</span>.<span class="w"> </span>Loss:<span class="w"> </span><span class="m">0</span>.4086027443408966.<span class="w"> </span>Accuracy:<span class="w"> </span><span class="m">94</span>
+Iteration:<span class="w"> </span><span class="m">2000</span>.<span class="w"> </span>Loss:<span class="w"> </span><span class="m">0</span>.09390712529420853.<span class="w"> </span>Accuracy:<span class="w"> </span><span class="m">95</span>
+Iteration:<span class="w"> </span><span class="m">2500</span>.<span class="w"> </span>Loss:<span class="w"> </span><span class="m">0</span>.07138358801603317.<span class="w"> </span>Accuracy:<span class="w"> </span><span class="m">95</span>
+Iteration:<span class="w"> </span><span class="m">3000</span>.<span class="w"> </span>Loss:<span class="w"> </span><span class="m">0</span>.05396252125501633.<span class="w"> </span>Accuracy:<span class="w"> </span><span class="m">96</span>
 </code></pre></div>
 <h3 id="summary-of-results">Summary of Results<a class="headerlink" href="#summary-of-results" title="Permanent link">&para;</a></h3>
 <table>
@@ -3204,12 +3344,12 @@ <h3 id="steps_3">Steps<a class="headerlink" href="#steps_3" title="Permanent lin
             <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Iteration: </span><span class="si">{}</span><span class="s1">. Loss: </span><span class="si">{}</span><span class="s1">. Accuracy: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">iter</span><span class="p">,</span> <span class="n">loss</span><span class="o">.</span><span class="n">item</span><span class="p">(),</span> <span class="n">accuracy</span><span class="p">))</span>
 </code></pre></div>
 </div>
-<div class="highlight"><pre><span></span><code>Iteration: <span class="m">500</span>. Loss: <span class="m">0</span>.36831170320510864. Accuracy: <span class="m">88</span>
-Iteration: <span class="m">1000</span>. Loss: <span class="m">0</span>.31790846586227417. Accuracy: <span class="m">92</span>
-Iteration: <span class="m">1500</span>. Loss: <span class="m">0</span>.1510857343673706. Accuracy: <span class="m">94</span>
-Iteration: <span class="m">2000</span>. Loss: <span class="m">0</span>.08368007838726044. Accuracy: <span class="m">95</span>
-Iteration: <span class="m">2500</span>. Loss: <span class="m">0</span>.13419771194458008. Accuracy: <span class="m">96</span>
-Iteration: <span class="m">3000</span>. Loss: <span class="m">0</span>.16750787198543549. Accuracy: <span class="m">96</span>
+<div class="highlight"><pre><span></span><code>Iteration:<span class="w"> </span><span class="m">500</span>.<span class="w"> </span>Loss:<span class="w"> </span><span class="m">0</span>.36831170320510864.<span class="w"> </span>Accuracy:<span class="w"> </span><span class="m">88</span>
+Iteration:<span class="w"> </span><span class="m">1000</span>.<span class="w"> </span>Loss:<span class="w"> </span><span class="m">0</span>.31790846586227417.<span class="w"> </span>Accuracy:<span class="w"> </span><span class="m">92</span>
+Iteration:<span class="w"> </span><span class="m">1500</span>.<span class="w"> </span>Loss:<span class="w"> </span><span class="m">0</span>.1510857343673706.<span class="w"> </span>Accuracy:<span class="w"> </span><span class="m">94</span>
+Iteration:<span class="w"> </span><span class="m">2000</span>.<span class="w"> </span>Loss:<span class="w"> </span><span class="m">0</span>.08368007838726044.<span class="w"> </span>Accuracy:<span class="w"> </span><span class="m">95</span>
+Iteration:<span class="w"> </span><span class="m">2500</span>.<span class="w"> </span>Loss:<span class="w"> </span><span class="m">0</span>.13419771194458008.<span class="w"> </span>Accuracy:<span class="w"> </span><span class="m">96</span>
+Iteration:<span class="w"> </span><span class="m">3000</span>.<span class="w"> </span>Loss:<span class="w"> </span><span class="m">0</span>.16750787198543549.<span class="w"> </span>Accuracy:<span class="w"> </span><span class="m">96</span>
 </code></pre></div>
 <div class="admonition tip">
 <p class="admonition-title">More Efficient Convolutions via Toeplitz Matrices</p>
@@ -3351,7 +3491,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -3414,12 +3554,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/index.html b/deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/index.html
index a5c67c856..98923cf13 100644
--- a/deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/index.html
+++ b/deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -1150,11 +1166,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1182,14 +1322,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1242,14 +1382,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1380,14 +1520,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1403,14 +1543,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1463,14 +1603,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -2373,7 +2513,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -2436,12 +2576,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/index.html b/deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/index.html
index 146300c0e..17dff2e21 100644
--- a/deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/index.html
+++ b/deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -1220,11 +1236,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1252,14 +1392,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1312,14 +1452,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1450,14 +1590,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1473,14 +1613,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1533,14 +1673,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -3420,7 +3560,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -3483,12 +3623,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/practical_pytorch/pytorch_gradients/index.html b/deep_learning/practical_pytorch/pytorch_gradients/index.html
index 765d7d95b..4f7de3df0 100644
--- a/deep_learning/practical_pytorch/pytorch_gradients/index.html
+++ b/deep_learning/practical_pytorch/pytorch_gradients/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_gradients/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -980,11 +996,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1012,14 +1152,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1072,14 +1212,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1210,14 +1350,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1233,14 +1373,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1293,14 +1433,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1769,7 +1909,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1832,12 +1972,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/practical_pytorch/pytorch_linear_regression/index.html b/deep_learning/practical_pytorch/pytorch_linear_regression/index.html
index 8e99d687e..233864d09 100644
--- a/deep_learning/practical_pytorch/pytorch_linear_regression/index.html
+++ b/deep_learning/practical_pytorch/pytorch_linear_regression/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_linear_regression/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -1042,11 +1058,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1074,14 +1214,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1134,14 +1274,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1272,14 +1412,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1295,14 +1435,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1355,14 +1495,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -2476,7 +2616,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -2539,12 +2679,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/practical_pytorch/pytorch_logistic_regression/index.html b/deep_learning/practical_pytorch/pytorch_logistic_regression/index.html
index 015b3b040..f2f76687f 100644
--- a/deep_learning/practical_pytorch/pytorch_logistic_regression/index.html
+++ b/deep_learning/practical_pytorch/pytorch_logistic_regression/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_logistic_regression/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -1192,11 +1208,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1224,14 +1364,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1284,14 +1424,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1422,14 +1562,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1445,14 +1585,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1505,14 +1645,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -3571,7 +3711,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -3634,12 +3774,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/index.html b/deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/index.html
index cf39312cd..c8dce8a52 100644
--- a/deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/index.html
+++ b/deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -1237,11 +1253,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1269,14 +1409,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1329,14 +1469,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1467,14 +1607,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1490,14 +1630,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1550,14 +1690,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -3271,7 +3411,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -3334,12 +3474,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/practical_pytorch/pytorch_matrices/index.html b/deep_learning/practical_pytorch/pytorch_matrices/index.html
index af45f1cc5..6121c9378 100644
--- a/deep_learning/practical_pytorch/pytorch_matrices/index.html
+++ b/deep_learning/practical_pytorch/pytorch_matrices/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_matrices/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -1104,11 +1120,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1136,14 +1276,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1196,14 +1336,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1334,14 +1474,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1357,14 +1497,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1417,14 +1557,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -2131,34 +2271,34 @@ <h4 id="numpy-to-torch">NumPy to Torch<a class="headerlink" href="#numpy-to-torc
 <table>
 <thead>
 <tr>
-<th align="center">NumPy Array Type</th>
-<th align="center">Torch Tensor Type</th>
+<th style="text-align: center;">NumPy Array Type</th>
+<th style="text-align: center;">Torch Tensor Type</th>
 </tr>
 </thead>
 <tbody>
 <tr>
-<td align="center">int64</td>
-<td align="center">LongTensor</td>
+<td style="text-align: center;">int64</td>
+<td style="text-align: center;">LongTensor</td>
 </tr>
 <tr>
-<td align="center">int32</td>
-<td align="center">IntegerTensor</td>
+<td style="text-align: center;">int32</td>
+<td style="text-align: center;">IntegerTensor</td>
 </tr>
 <tr>
-<td align="center">uint8</td>
-<td align="center">ByteTensor</td>
+<td style="text-align: center;">uint8</td>
+<td style="text-align: center;">ByteTensor</td>
 </tr>
 <tr>
-<td align="center">float64</td>
-<td align="center">DoubleTensor</td>
+<td style="text-align: center;">float64</td>
+<td style="text-align: center;">DoubleTensor</td>
 </tr>
 <tr>
-<td align="center">float32</td>
-<td align="center">FloatTensor</td>
+<td style="text-align: center;">float32</td>
+<td style="text-align: center;">FloatTensor</td>
 </tr>
 <tr>
-<td align="center">double</td>
-<td align="center">DoubleTensor</td>
+<td style="text-align: center;">double</td>
+<td style="text-align: center;">DoubleTensor</td>
 </tr>
 </tbody>
 </table>
@@ -2624,7 +2764,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -2687,12 +2827,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/index.html b/deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/index.html
index 59ca2c12a..6e3a0e2fa 100644
--- a/deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/index.html
+++ b/deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -1170,11 +1186,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1202,14 +1342,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1262,14 +1402,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1400,14 +1540,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1423,14 +1563,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1483,14 +1623,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -3070,7 +3210,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -3133,12 +3273,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/production_pytorch/speed_optimization_basics_numba/index.html b/deep_learning/production_pytorch/speed_optimization_basics_numba/index.html
index ada4b2f4a..c602a5700 100644
--- a/deep_learning/production_pytorch/speed_optimization_basics_numba/index.html
+++ b/deep_learning/production_pytorch/speed_optimization_basics_numba/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/production_pytorch/speed_optimization_basics_numba/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -1008,11 +1024,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -1040,14 +1180,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1100,14 +1240,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1238,14 +1378,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1261,14 +1401,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1321,14 +1461,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1728,7 +1868,7 @@ <h2 id="__comments">Comments</h2>
       
       
         
-        <a href="../../../machine_learning/intro/" class="md-footer__link md-footer__link--next" aria-label="Next: Introduction" rel="next">
+        <a href="../../../language_model/intro/" class="md-footer__link md-footer__link--next" aria-label="Next: Introduction" rel="next">
           <div class="md-footer__title">
             <div class="md-ellipsis">
               <span class="md-footer__direction">
@@ -1749,7 +1889,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1812,12 +1952,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/deep_learning/readings/index.html b/deep_learning/readings/index.html
index d9ff01e02..9ef19879f 100644
--- a/deep_learning/readings/index.html
+++ b/deep_learning/readings/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/deep_learning/readings/">
       
       <link rel="icon" href="../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -355,7 +371,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -914,11 +930,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -946,14 +1086,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1006,14 +1146,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1144,14 +1284,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1167,14 +1307,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1227,14 +1367,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1696,7 +1836,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1759,12 +1899,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/home/index.html b/home/index.html
index 7fc3a6dea..350021d05 100644
--- a/home/index.html
+++ b/home/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/home/">
       
       <link rel="icon" href="../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -928,11 +944,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -960,14 +1100,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1020,14 +1160,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1158,14 +1298,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1181,14 +1321,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1241,14 +1381,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1452,7 +1592,7 @@
                 
                 <h1 id="learn-the-whole-ai-pipeline-with">Learn The Whole AI Pipeline With<a class="headerlink" href="#learn-the-whole-ai-pipeline-with" title="Permanent link">&para;</a></h1>
 <p>You will the learn the suite of tools to build an end-to-end deep learning pipeline.</p>
-<div class="tabbed-set" data-tabs="1:1"><input checked="checked" id="__tabbed_1_1" name="__tabbed_1" type="radio" /><label for="__tabbed_1_1">Tools You Will Learn To Use</label><div class="tabbed-content">
+<div class="tabbed-set" data-tabs="1:1"><input checked="checked" id="learn-the-whole-ai-pipeline-with-tools-you-will-learn-to-use" name="__tabbed_1" type="radio" /><label for="learn-the-whole-ai-pipeline-with-tools-you-will-learn-to-use">Tools You Will Learn To Use</label><div class="tabbed-content">
 <ul>
 <li>PyTorch (CPU/GPU)</li>
 <li>Scikit-learn (CPU)</li>
@@ -1549,7 +1689,7 @@ <h1 id="learn-the-whole-ai-pipeline-with">Learn The Whole AI Pipeline With<a cla
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1612,12 +1752,16 @@ <h1 id="learn-the-whole-ai-pipeline-with">Learn The Whole AI Pipeline With<a cla
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/index.html b/index.html
index abdb7dd27..b8be693f3 100644
--- a/index.html
+++ b/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/">
       
       <link rel="icon" href="docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -90,15 +90,27 @@
           <div class="md-banner__inner md-announce__inner md-grid md-typeset">
             
   For updates follow us on
-  <a href="https://twitter.com/ritchieng">
+  <a href="https://twitter.com/deeplearningwiz">
     <span class="twemoji twitter">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M459.37 151.716c.325 4.548.325 9.097.325 13.645 0 138.72-105.583 298.558-298.558 298.558-59.452 0-114.68-17.219-161.137-47.106 8.447.974 16.568 1.299 25.34 1.299 49.055 0 94.213-16.568 130.274-44.832-46.132-.975-84.792-31.188-98.112-72.772 6.498.974 12.995 1.624 19.818 1.624 9.421 0 18.843-1.3 27.614-3.573-48.081-9.747-84.143-51.98-84.143-102.985v-1.299c13.969 7.797 30.214 12.67 47.431 13.319-28.264-18.843-46.781-51.005-46.781-87.391 0-19.492 5.197-37.36 14.294-52.954 51.655 63.675 129.3 105.258 216.365 109.807-1.624-7.797-2.599-15.918-2.599-24.04 0-57.828 46.782-104.934 104.934-104.934 30.213 0 57.502 12.67 76.67 33.137 23.715-4.548 46.456-13.32 66.599-25.34-7.798 24.366-24.366 44.833-46.132 57.827 21.117-2.273 41.584-8.122 60.426-16.243-14.292 20.791-32.161 39.308-52.628 54.253z"/></svg>
     </span>
     <strong>Twitter</strong>
   </a>
+  <a href="https://www.linkedin.com/company/deeplearningwizard">
+    <span class="twemoji linkedin" style="color:#0077B5">
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M416 32H31.9C14.3 32 0 46.5 0 64.3v383.4C0 465.5 14.3 480 31.9 480H416c17.6 0 32-14.5 32-32.3V64.3c0-17.8-14.4-32.3-32-32.3zM135.4 416H69V202.2h66.5V416zm-33.2-243c-21.3 0-38.5-17.3-38.5-38.5S80.9 96 102.2 96c21.2 0 38.5 17.3 38.5 38.5 0 21.3-17.2 38.5-38.5 38.5zm282.1 243h-66.4V312c0-24.8-.5-56.7-34.5-56.7-34.6 0-39.9 27-39.9 54.9V416h-66.4V202.2h63.7v29.2h.9c8.9-16.8 30.6-34.5 62.9-34.5 67.2 0 79.7 44.3 79.7 101.9V416z"/></svg>
+    </span>
+    <strong>Linkedin</strong>
+  </a>
+  <a href="https://www.facebook.com/DeepLearningWizard/">
+    <span class="twemoji facebook" style="color:#4267B2">
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M504 256C504 119 393 8 256 8S8 119 8 256c0 123.78 90.69 226.38 209.25 245V327.69h-63V256h63v-54.64c0-62.15 37-96.48 93.67-96.48 27.14 0 55.52 4.84 55.52 4.84v61h-31.28c-30.8 0-40.41 19.12-40.41 38.73V256h68.78l-11 71.69h-57.78V501C413.31 482.38 504 379.78 504 256z"/></svg>
+    </span>
+    <strong>Facebook</strong>
+  </a>
   or subscribe to our channel on
   <a href="https://www.youtube.com/channel/UCJz2MIjiCosOQCwhnsYxeEw">
-    <span class="twemoji twitter" style="color:red">
+    <span class="twemoji twitter" style="color:#FF0000">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512"><path d="M549.655 124.083c-6.281-23.65-24.787-42.276-48.284-48.597C458.781 64 288 64 288 64S117.22 64 74.629 75.486c-23.497 6.322-42.003 24.947-48.284 48.597-11.412 42.867-11.412 132.305-11.412 132.305s0 89.438 11.412 132.305c6.281 23.65 24.787 41.5 48.284 47.821C117.22 448 288 448 288 448s170.78 0 213.371-11.486c23.497-6.321 42.003-24.171 48.284-47.821 11.412-42.867 11.412-132.305 11.412-132.305s0-89.438-11.412-132.305zm-317.51 213.508V175.185l142.739 81.205-142.739 81.201z"/></svg>
     </span>
     <strong>YouTube</strong>
@@ -204,7 +216,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -282,6 +294,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -398,7 +426,7 @@ <h1>A less painful path to deep learning familiarity</h1>
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -965,11 +993,135 @@ <h1>A less painful path to deep learning familiarity</h1>
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -997,14 +1149,14 @@ <h1>A less painful path to deep learning familiarity</h1>
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1057,14 +1209,14 @@ <h1>A less painful path to deep learning familiarity</h1>
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1195,14 +1347,14 @@ <h1>A less painful path to deep learning familiarity</h1>
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1218,14 +1370,14 @@ <h1>A less painful path to deep learning familiarity</h1>
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1278,14 +1430,14 @@ <h1>A less painful path to deep learning familiarity</h1>
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1484,13 +1636,17 @@ <h1>A less painful path to deep learning familiarity</h1>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": ".", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": ".", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
   
       <script src="assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   <script src="overrides/assets/javascripts/bundle.a07ad7e8.min.js"></script>
diff --git a/language_model/containers/hpc_containers_apptainer/index.html b/language_model/containers/hpc_containers_apptainer/index.html
new file mode 100644
index 000000000..46691d2b4
--- /dev/null
+++ b/language_model/containers/hpc_containers_apptainer/index.html
@@ -0,0 +1,2344 @@
+
+<!doctype html>
+<html lang="en" class="no-js">
+  <head>
+    
+      <meta charset="utf-8">
+      <meta name="viewport" content="width=device-width,initial-scale=1">
+      
+        <meta name="description" content="We try to make learning deep learning, deep bayesian learning, and deep reinforcement learning math and code easier. Used by thousands.">
+      
+      
+      
+        <meta name="author" content="Ritchie Ng">
+      
+      
+        <link rel="canonical" href="https://www.deeplearningwizard.com/language_model/containers/hpc_containers_apptainer/">
+      
+      <link rel="icon" href="../../../docs/assets/favicon.ico">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
+    
+    
+      
+        <title>HPC Containers with Apptainer - Deep Learning Wizard</title>
+      
+    
+    
+      <link rel="stylesheet" href="../../../assets/stylesheets/main.f7f47774.min.css">
+      
+        
+        <link rel="stylesheet" href="../../../assets/stylesheets/palette.3f5d1f46.min.css">
+        
+      
+    
+    
+    
+      
+        
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700%7CRoboto+Mono&display=fallback">
+        <style>:root{--md-text-font-family:"Roboto";--md-code-font-family:"Roboto Mono"}</style>
+      
+    
+    
+    
+    
+      
+
+  
+
+
+  
+
+  
+
+
+  <script>window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)},ga.l=+new Date,ga("create","UA-122083328-1","auto"),ga("set","anonymizeIp",!0),ga("send","pageview"),document.addEventListener("DOMContentLoaded",function(){document.forms.search&&document.forms.search.query.addEventListener("blur",function(){var e;this.value&&(e=document.location.pathname,ga("send","pageview",e+"?q="+this.value))}),"undefined"!=typeof location$&&location$.subscribe(function(e){ga("send","pageview",e.pathname)})})</script>
+  <script async src="https://www.google-analytics.com/analytics.js"></script>
+
+
+    
+    
+  </head>
+  
+  
+    
+    
+      
+    
+    
+    
+    
+    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo">
+  
+    
+    <script>function __prefix(e){return new URL("../../..",location).pathname+"."+e}function __get(e,t=localStorage){return JSON.parse(t.getItem(__prefix(e)))}</script>
+    
+      <script>var palette=__get("__palette");if(null!==palette&&"object"==typeof palette.color)for(var key in palette.color)document.body.setAttribute("data-md-color-"+key,palette.color[key])</script>
+    
+    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
+    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
+    <label class="md-overlay" for="__drawer"></label>
+    <div data-md-component="skip">
+      
+        
+        <a href="#containers" class="md-skip">
+          Skip to content
+        </a>
+      
+    </div>
+    <div data-md-component="announce">
+      
+    </div>
+    
+      <header class="md-header" data-md-component="header">
+  <nav class="md-header__inner md-grid" aria-label="Header">
+    <a href="../../.." title="Deep Learning Wizard" class="md-header__button md-logo" aria-label="Deep Learning Wizard" data-md-component="logo">
+      
+  
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M396.8 352h22.4c6.4 0 12.8-6.4 12.8-12.8V108.8c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v230.4c0 6.4 6.4 12.8 12.8 12.8zm-192 0h22.4c6.4 0 12.8-6.4 12.8-12.8V140.8c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v198.4c0 6.4 6.4 12.8 12.8 12.8zm96 0h22.4c6.4 0 12.8-6.4 12.8-12.8V204.8c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v134.4c0 6.4 6.4 12.8 12.8 12.8zM496 400H48V80c0-8.84-7.16-16-16-16H16C7.16 64 0 71.16 0 80v336c0 17.67 14.33 32 32 32h464c8.84 0 16-7.16 16-16v-16c0-8.84-7.16-16-16-16zm-387.2-48h22.4c6.4 0 12.8-6.4 12.8-12.8v-70.4c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v70.4c0 6.4 6.4 12.8 12.8 12.8z"/></svg>
+
+    </a>
+    <label class="md-header__button md-icon" for="__drawer">
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2z"/></svg>
+    </label>
+    <div class="md-header__title" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            Deep Learning Wizard
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+            
+              HPC Containers with Apptainer
+            
+          </span>
+        </div>
+      </div>
+    </div>
+    
+      <form class="md-header__option" data-md-component="palette">
+        
+          
+          
+          <input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo"  aria-label="Switch to dark mode"  type="radio" name="__palette" id="__palette_1">
+          
+            <label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_2" hidden>
+              <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M7 10a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2m10-3a5 5 0 0 1 5 5 5 5 0 0 1-5 5H7a5 5 0 0 1-5-5 5 5 0 0 1 5-5h10M7 9a3 3 0 0 0-3 3 3 3 0 0 0 3 3h10a3 3 0 0 0 3-3 3 3 0 0 0-3-3H7z"/></svg>
+            </label>
+          
+        
+          
+          
+          <input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="indigo" data-md-color-accent="indigo"  aria-label="Switch to light mode"  type="radio" name="__palette" id="__palette_2">
+          
+            <label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
+              <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 7H7a5 5 0 0 0-5 5 5 5 0 0 0 5 5h10a5 5 0 0 0 5-5 5 5 0 0 0-5-5m0 8a3 3 0 0 1-3-3 3 3 0 0 1 3-3 3 3 0 0 1 3 3 3 3 0 0 1-3 3z"/></svg>
+            </label>
+          
+        
+      </form>
+    
+    
+    
+      <label class="md-header__button md-icon" for="__search">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5z"/></svg>
+      </label>
+      
+<div class="md-search" data-md-component="search" role="dialog">
+  <label class="md-search__overlay" for="__search"></label>
+  <div class="md-search__inner" role="search">
+    <form class="md-search__form" name="search">
+      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
+      <label class="md-search__icon md-icon" for="__search">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5z"/></svg>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12z"/></svg>
+      </label>
+      <nav class="md-search__options" aria-label="Search">
+        
+          <a href="javascript:void(0)" class="md-search__icon md-icon" aria-label="Share" data-clipboard data-clipboard-text="" data-md-component="search-share" tabindex="-1">
+            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M18 16.08c-.76 0-1.44.3-1.96.77L8.91 12.7c.05-.23.09-.46.09-.7 0-.24-.04-.47-.09-.7l7.05-4.11c.54.5 1.25.81 2.04.81a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3c0 .24.04.47.09.7L8.04 9.81C7.5 9.31 6.79 9 6 9a3 3 0 0 0-3 3 3 3 0 0 0 3 3c.79 0 1.5-.31 2.04-.81l7.12 4.15c-.05.21-.08.43-.08.66 0 1.61 1.31 2.91 2.92 2.91 1.61 0 2.92-1.3 2.92-2.91A2.92 2.92 0 0 0 18 16.08z"/></svg>
+          </a>
+        
+        <button type="reset" class="md-search__icon md-icon" aria-label="Clear" tabindex="-1">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41z"/></svg>
+        </button>
+      </nav>
+      
+        <div class="md-search__suggest" data-md-component="search-suggest"></div>
+      
+    </form>
+    <div class="md-search__output">
+      <div class="md-search__scrollwrap" data-md-scrollfix>
+        <div class="md-search-result" data-md-component="search-result">
+          <div class="md-search-result__meta">
+            Initializing search
+          </div>
+          <ol class="md-search-result__list"></ol>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+    
+    
+      <div class="md-header__source">
+        
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+  </div>
+  <div class="md-source__repository">
+    ritchieng/deep-learning-wizard
+  </div>
+</a>
+      </div>
+    
+  </nav>
+</header>
+    
+    <div class="md-container" data-md-component="container">
+      
+      
+        
+          
+<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
+  <div class="md-tabs__inner md-grid">
+    <ul class="md-tabs__list">
+      
+        
+  
+  
+
+
+  <li class="md-tabs__item">
+    <a href="../../.." class="md-tabs__link">
+      Home
+    </a>
+  </li>
+
+      
+        
+  
+  
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../home/" class="md-tabs__link">
+        About
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../deep_learning/intro/" class="md-tabs__link">
+        Deep Learning
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+    
+  
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../intro/" class="md-tabs__link md-tabs__link--active">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../machine_learning/intro/" class="md-tabs__link">
+        Machine Learning
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../programming/intro/" class="md-tabs__link">
+        Programming
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
+    
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../data_engineering/nosql/cassandra/intro/" class="md-tabs__link">
+        Data Engineering
+      </a>
+    </li>
+  
+
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../news/news/" class="md-tabs__link">
+        News
+      </a>
+    </li>
+  
+
+      
+    </ul>
+  </div>
+</nav>
+        
+      
+      <main class="md-main" data-md-component="main">
+        <div class="md-main__inner md-grid">
+          
+            
+              
+              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+                    
+
+  
+
+
+<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0">
+  <label class="md-nav__title" for="__drawer">
+    <a href="../../.." title="Deep Learning Wizard" class="md-nav__button md-logo" aria-label="Deep Learning Wizard" data-md-component="logo">
+      
+  
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M396.8 352h22.4c6.4 0 12.8-6.4 12.8-12.8V108.8c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v230.4c0 6.4 6.4 12.8 12.8 12.8zm-192 0h22.4c6.4 0 12.8-6.4 12.8-12.8V140.8c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v198.4c0 6.4 6.4 12.8 12.8 12.8zm96 0h22.4c6.4 0 12.8-6.4 12.8-12.8V204.8c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v134.4c0 6.4 6.4 12.8 12.8 12.8zM496 400H48V80c0-8.84-7.16-16-16-16H16C7.16 64 0 71.16 0 80v336c0 17.67 14.33 32 32 32h464c8.84 0 16-7.16 16-16v-16c0-8.84-7.16-16-16-16zm-387.2-48h22.4c6.4 0 12.8-6.4 12.8-12.8v-70.4c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v70.4c0 6.4 6.4 12.8 12.8 12.8z"/></svg>
+
+    </a>
+    Deep Learning Wizard
+  </label>
+  
+    <div class="md-nav__source">
+      
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+  </div>
+  <div class="md-source__repository">
+    ritchieng/deep-learning-wizard
+  </div>
+</a>
+    </div>
+  
+  <ul class="md-nav__list" data-md-scrollfix>
+    
+      
+      
+      
+
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../.." class="md-nav__link">
+        Home
+      </a>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_2" type="checkbox" id="__nav_2" >
+      
+      <label class="md-nav__link" for="__nav_2">
+        About
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="About" data-md-level="1">
+        <label class="md-nav__title" for="__nav_2">
+          <span class="md-nav__icon md-icon"></span>
+          About
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../home/" class="md-nav__link">
+        Intro
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../about/" class="md-nav__link">
+        About Us
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../review/" class="md-nav__link">
+        Reviews
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../consultancy/" class="md-nav__link">
+        Consultancy
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3" type="checkbox" id="__nav_3" >
+      
+      <label class="md-nav__link" for="__nav_3">
+        Deep Learning
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Deep Learning" data-md-level="1">
+        <label class="md-nav__title" for="__nav_3">
+          <span class="md-nav__icon md-icon"></span>
+          Deep Learning
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/course_progression/" class="md-nav__link">
+        Course Progression
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3_3" type="checkbox" id="__nav_3_3" >
+      
+      <label class="md-nav__link" for="__nav_3_3">
+        Practical Deep Learning with PyTorch
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Practical Deep Learning with PyTorch" data-md-level="2">
+        <label class="md-nav__title" for="__nav_3_3">
+          <span class="md-nav__icon md-icon"></span>
+          Practical Deep Learning with PyTorch
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/practical_pytorch/pytorch_matrices/" class="md-nav__link">
+        Matrices
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/practical_pytorch/pytorch_gradients/" class="md-nav__link">
+        Gradients
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/practical_pytorch/pytorch_linear_regression/" class="md-nav__link">
+        Linear Regression
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/practical_pytorch/pytorch_logistic_regression/" class="md-nav__link">
+        Logistic Regression
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/" class="md-nav__link">
+        Feedforward Neural Networks (FNN)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/" class="md-nav__link">
+        Convolutional Neural Networks (CNN)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/" class="md-nav__link">
+        Recurrent Neural Networks (RNN)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/" class="md-nav__link">
+        Long Short Term Memory Neural Networks (LSTM)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/practical_pytorch/pytorch_autoencoder/" class="md-nav__link">
+        Autoencoders (AE)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/" class="md-nav__link">
+        Fully-connected Overcomplete Autoencoder (AE)
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3_4" type="checkbox" id="__nav_3_4" >
+      
+      <label class="md-nav__link" for="__nav_3_4">
+        Improving Deep Learning with PyTorch
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Improving Deep Learning with PyTorch" data-md-level="2">
+        <label class="md-nav__title" for="__nav_3_4">
+          <span class="md-nav__icon md-icon"></span>
+          Improving Deep Learning with PyTorch
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/" class="md-nav__link">
+        Derivative, Gradient and Jacobian
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/boosting_models_pytorch/forwardpropagation_backpropagation_gradientdescent/" class="md-nav__link">
+        Forward- and Backward-propagation and Gradient Descent (From Scratch FNN Regression)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/boosting_models_pytorch/lr_scheduling/" class="md-nav__link">
+        Learning Rate Scheduling
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/boosting_models_pytorch/optimizers/" class="md-nav__link">
+        Optimization Algorithms
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/" class="md-nav__link">
+        Weight Initialization and Activation Functions
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3_5" type="checkbox" id="__nav_3_5" >
+      
+      <label class="md-nav__link" for="__nav_3_5">
+        Deep Reinforcement Learning with PyTorch
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Deep Reinforcement Learning with PyTorch" data-md-level="2">
+        <label class="md-nav__title" for="__nav_3_5">
+          <span class="md-nav__icon md-icon"></span>
+          Deep Reinforcement Learning with PyTorch
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/deep_reinforcement_learning_pytorch/supervised_to_rl/" class="md-nav__link">
+        Supervised Learning to Reinforcement Learning (RL)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/" class="md-nav__link">
+        Markov Decision Processes (MDP) and Bellman Equations
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/" class="md-nav__link">
+        Dynamic Programming
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3_6" type="checkbox" id="__nav_3_6" >
+      
+      <label class="md-nav__link" for="__nav_3_6">
+        From Scratch Deep Learning with Python/PyTorch
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="From Scratch Deep Learning with Python/PyTorch" data-md-level="2">
+        <label class="md-nav__title" for="__nav_3_6">
+          <span class="md-nav__icon md-icon"></span>
+          From Scratch Deep Learning with Python/PyTorch
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/fromscratch/fromscratch_logistic_regression/" class="md-nav__link">
+        From Scratch Logistic Regression Classification
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3_7" type="checkbox" id="__nav_3_7" >
+      
+      <label class="md-nav__link" for="__nav_3_7">
+        Compute Optimization
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Compute Optimization" data-md-level="2">
+        <label class="md-nav__title" for="__nav_3_7">
+          <span class="md-nav__icon md-icon"></span>
+          Compute Optimization
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/production_pytorch/speed_optimization_basics_numba/" class="md-nav__link">
+        Speed Optimization Basics Numba
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+    
+  
+  
+    
+    <li class="md-nav__item md-nav__item--active md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" checked>
+      
+      <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+    
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" checked>
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+    
+  
+  
+    <li class="md-nav__item md-nav__item--active">
+      
+      <input class="md-nav__toggle md-toggle" data-md-toggle="toc" type="checkbox" id="__toc">
+      
+      
+        
+      
+      
+        <label class="md-nav__link md-nav__link--active" for="__toc">
+          HPC Containers with Apptainer
+          <span class="md-nav__icon md-icon"></span>
+        </label>
+      
+      <a href="./" class="md-nav__link md-nav__link--active">
+        HPC Containers with Apptainer
+      </a>
+      
+        
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+  
+  
+  
+    
+  
+  
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+      
+        <li class="md-nav__item">
+  <a href="#benefits-of-apptainer" class="md-nav__link">
+    Benefits of Apptainer
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#installing-apptainer" class="md-nav__link">
+    Installing Apptainer
+  </a>
+  
+    <nav class="md-nav" aria-label="Installing Apptainer">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#install-apptainer" class="md-nav__link">
+    Install Apptainer
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#install-nvidia-libraries" class="md-nav__link">
+    Install NVIDIA libraries
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#set-nvidia-container-cli-apptainer-paths-deprecated" class="md-nav__link">
+    Set NVIDIA Container CLI Apptainer Paths (Deprecated)
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#how-to-use-apptainer-in-cpugpu-mode" class="md-nav__link">
+    How to Use Apptainer in CPU/GPU Mode
+  </a>
+  
+    <nav class="md-nav" aria-label="How to Use Apptainer in CPU/GPU Mode">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#option-a-transparent-image-container-workflow" class="md-nav__link">
+    Option A: Transparent Image Container Workflow
+  </a>
+  
+    <nav class="md-nav" aria-label="Option A: Transparent Image Container Workflow">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#1-build-container-sif" class="md-nav__link">
+    1. Build container (.sif)
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#2-run-container" class="md-nav__link">
+    2. Run container
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#notes-on-gpu-availability" class="md-nav__link">
+    Notes on GPU Availability
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#option-b-blackbox" class="md-nav__link">
+    Option B: Blackbox
+  </a>
+  
+    <nav class="md-nav" aria-label="Option B: Blackbox">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#1-build-container-sif_1" class="md-nav__link">
+    1. Build container (.sif)
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#2-convert-container-to-mutable-sandbox" class="md-nav__link">
+    2. Convert container to mutable sandbox
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#3-shell-into-writable-sandbox" class="md-nav__link">
+    3. Shell into writable sandbox
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#4-convert-container-to-immutable-image-sif" class="md-nav__link">
+    4. Convert container to immutable image (.sif)
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#basic-apptainer-commands-cheatsheet" class="md-nav__link">
+    Basic Apptainer Commands Cheatsheet
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#available-containers-definition-files" class="md-nav__link">
+    Available Containers &amp; Definition Files
+  </a>
+  
+    <nav class="md-nav" aria-label="Available Containers &amp; Definition Files">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#gpu-containers" class="md-nav__link">
+    GPU Containers
+  </a>
+  
+    <nav class="md-nav" aria-label="GPU Containers">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#ollama-mistral-7b-workloads" class="md-nav__link">
+    Ollama (Mistral 7b) Workloads
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#ollama-gemma-workloads" class="md-nav__link">
+    Ollama (Gemma) Workloads
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#llamaindex-workloads" class="md-nav__link">
+    LLamaindex workloads
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#cpu-containers" class="md-nav__link">
+    CPU Containers
+  </a>
+  
+    <nav class="md-nav" aria-label="CPU Containers">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#math-workloads" class="md-nav__link">
+    Math Workloads
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#summary" class="md-nav__link">
+    Summary
+  </a>
+  
+</li>
+      
+    </ul>
+  
+</nav>
+      
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
+        Machine Learning
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
+        <label class="md-nav__title" for="__nav_5">
+          <span class="md-nav__icon md-icon"></span>
+          Machine Learning
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../machine_learning/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
+      
+      <label class="md-nav__link" for="__nav_5_2">
+        RAPIDS cuDF
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
+        <label class="md-nav__title" for="__nav_5_2">
+          <span class="md-nav__icon md-icon"></span>
+          RAPIDS cuDF
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../machine_learning/gpu/rapids_cudf/" class="md-nav__link">
+        GPU DataFrames
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../machine_learning/gpu/gpu_fractional_differencing/" class="md-nav__link">
+        GPU/CPU Fractional Differencing
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+      
+      <label class="md-nav__link" for="__nav_6">
+        Programming
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Programming" data-md-level="1">
+        <label class="md-nav__title" for="__nav_6">
+          <span class="md-nav__icon md-icon"></span>
+          Programming
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../programming/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../programming/cpp/cpp/" class="md-nav__link">
+        C++
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../programming/bash/bash/" class="md-nav__link">
+        Bash
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../programming/python/python/" class="md-nav__link">
+        Python
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../programming/r/r/" class="md-nav__link">
+        R
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../programming/javascript/javascript/" class="md-nav__link">
+        Javascript
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../programming/electron/electron/" class="md-nav__link">
+        Electron
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../programming/sympy/calculus_sympy/" class="md-nav__link">
+        Sympy
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../programming/numpycupy/linalg_numpy_cupy/" class="md-nav__link">
+        NumPy and CuPy
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+      
+      <label class="md-nav__link" for="__nav_7">
+        Data Engineering
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
+        <label class="md-nav__title" for="__nav_7">
+          <span class="md-nav__icon md-icon"></span>
+          Data Engineering
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
+      
+      <label class="md-nav__link" for="__nav_7_1">
+        Cassandra (NoSQL)
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
+        <label class="md-nav__title" for="__nav_7_1">
+          <span class="md-nav__icon md-icon"></span>
+          Cassandra (NoSQL)
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../data_engineering/nosql/cassandra/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../data_engineering/nosql/cassandra/setting_up_cluster/" class="md-nav__link">
+        Cassandra Cluster Setup
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
+      
+      <label class="md-nav__link" for="__nav_8">
+        News
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="News" data-md-level="1">
+        <label class="md-nav__title" for="__nav_8">
+          <span class="md-nav__icon md-icon"></span>
+          News
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/news/" class="md-nav__link">
+        Welcome
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/dbs_gpu_rapids_nvidia_ensemble_frac_diff/" class="md-nav__link">
+        Fractional Differencing with GPU (GFD), DBS and NVIDIA, September 2019
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/defence_and_science_technology_agency_dsta_nvidia_talk_2016_06/" class="md-nav__link">
+        Deep Learning Introduction, Defence and Science Technology Agency (DSTA) and NVIDIA, June 2019
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/detect_waterbone_debris_ai_for_social_good_icml_2019_06/" class="md-nav__link">
+        Oral Presentation for AI for Social Good Workshop ICML, June 2019
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/it_youth_leader_2019_03_11/" class="md-nav__link">
+        IT Youth Leader of The Year 2019, March 2019
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/ammi_facebook_google_recap_2018_11_21/" class="md-nav__link">
+        AMMI (AIMS) supported by Facebook and Google, November 2018
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/nanjing_next_nus_tsinghua_ai_finance_healthcare_2018_11_01/" class="md-nav__link">
+        NExT++ AI in Healthcare and Finance, Nanjing, November 2018
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/facebook_pytorch_devcon_recap_2018_10_02/" class="md-nav__link">
+        Recap of Facebook PyTorch Developer Conference, San Francisco, September 2018
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/facebook_pytorch_developer_conference_2018_09_05/" class="md-nav__link">
+        Facebook PyTorch Developer Conference, San Francisco, September 2018
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/nvidia_nus_mit_datathon_2018_07_05/" class="md-nav__link">
+        NUS-MIT-NUHS NVIDIA Image Recognition Workshop, Singapore, July 2018
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/deep_learning_wizard_1y_2018_06_01/" class="md-nav__link">
+        Featured on PyTorch Website 2018
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/nvidia_self_driving_cars_talk_2017_06_21/" class="md-nav__link">
+        NVIDIA Self Driving Cars & Healthcare Talk, Singapore, June 2017
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/deep_learning_wizard_nvidia_inception_2018_05_01/" class="md-nav__link">
+        NVIDIA Inception Partner Status, Singapore, May 2017
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+  </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+            
+            
+              
+              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+                    
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+  
+  
+  
+    
+  
+  
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+      
+        <li class="md-nav__item">
+  <a href="#benefits-of-apptainer" class="md-nav__link">
+    Benefits of Apptainer
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#installing-apptainer" class="md-nav__link">
+    Installing Apptainer
+  </a>
+  
+    <nav class="md-nav" aria-label="Installing Apptainer">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#install-apptainer" class="md-nav__link">
+    Install Apptainer
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#install-nvidia-libraries" class="md-nav__link">
+    Install NVIDIA libraries
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#set-nvidia-container-cli-apptainer-paths-deprecated" class="md-nav__link">
+    Set NVIDIA Container CLI Apptainer Paths (Deprecated)
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#how-to-use-apptainer-in-cpugpu-mode" class="md-nav__link">
+    How to Use Apptainer in CPU/GPU Mode
+  </a>
+  
+    <nav class="md-nav" aria-label="How to Use Apptainer in CPU/GPU Mode">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#option-a-transparent-image-container-workflow" class="md-nav__link">
+    Option A: Transparent Image Container Workflow
+  </a>
+  
+    <nav class="md-nav" aria-label="Option A: Transparent Image Container Workflow">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#1-build-container-sif" class="md-nav__link">
+    1. Build container (.sif)
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#2-run-container" class="md-nav__link">
+    2. Run container
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#notes-on-gpu-availability" class="md-nav__link">
+    Notes on GPU Availability
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#option-b-blackbox" class="md-nav__link">
+    Option B: Blackbox
+  </a>
+  
+    <nav class="md-nav" aria-label="Option B: Blackbox">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#1-build-container-sif_1" class="md-nav__link">
+    1. Build container (.sif)
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#2-convert-container-to-mutable-sandbox" class="md-nav__link">
+    2. Convert container to mutable sandbox
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#3-shell-into-writable-sandbox" class="md-nav__link">
+    3. Shell into writable sandbox
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#4-convert-container-to-immutable-image-sif" class="md-nav__link">
+    4. Convert container to immutable image (.sif)
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#basic-apptainer-commands-cheatsheet" class="md-nav__link">
+    Basic Apptainer Commands Cheatsheet
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#available-containers-definition-files" class="md-nav__link">
+    Available Containers &amp; Definition Files
+  </a>
+  
+    <nav class="md-nav" aria-label="Available Containers &amp; Definition Files">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#gpu-containers" class="md-nav__link">
+    GPU Containers
+  </a>
+  
+    <nav class="md-nav" aria-label="GPU Containers">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#ollama-mistral-7b-workloads" class="md-nav__link">
+    Ollama (Mistral 7b) Workloads
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#ollama-gemma-workloads" class="md-nav__link">
+    Ollama (Gemma) Workloads
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#llamaindex-workloads" class="md-nav__link">
+    LLamaindex workloads
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#cpu-containers" class="md-nav__link">
+    CPU Containers
+  </a>
+  
+    <nav class="md-nav" aria-label="CPU Containers">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#math-workloads" class="md-nav__link">
+    Math Workloads
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#summary" class="md-nav__link">
+    Summary
+  </a>
+  
+</li>
+      
+    </ul>
+  
+</nav>
+                  </div>
+                </div>
+              </div>
+            
+          
+          <div class="md-content" data-md-component="content">
+            <article class="md-content__inner md-typeset">
+              
+                
+                  <a href="https://github.com/ritchieng/deep-learning-wizard/edit/master/docs/language_model/containers/hpc_containers_apptainer.md" title="Edit this page" class="md-content__button md-icon">
+                    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20.71 7.04c.39-.39.39-1.04 0-1.41l-2.34-2.34c-.37-.39-1.02-.39-1.41 0l-1.84 1.83 3.75 3.75M3 17.25V21h3.75L17.81 9.93l-3.75-3.75L3 17.25z"/></svg>
+                  </a>
+                
+                
+                <h1 id="containers">Containers<a class="headerlink" href="#containers" title="Permanent link">&para;</a></h1>
+<p>To run LLMs easily from laptops and desktops to the cloud, we will be introducing <strong>Apptainer</strong> (formerly Singularity) to achieve this where it has largely been adopted by the community to execute HPC workloads locally and in HPC data centers.</p>
+<h2 id="benefits-of-apptainer">Benefits of Apptainer<a class="headerlink" href="#benefits-of-apptainer" title="Permanent link">&para;</a></h2>
+<p><a href="https://apptainer.org/">Apptainer</a> has quite a few benefits such as:</p>
+<ul>
+<li><strong>Mobility of Compute</strong>: It supports computational mobility through a single-file SIF container format.</li>
+<li><strong>Integration over Isolation</strong>: It emphasizes integration, utilizing GPUs, high-speed networks, and parallel filesystems by default. In particular, it provides native support for running application containers using NVIDIA's CUDA or AMD's ROCm, enabling easy access to GPU-enabled machine learning frameworks like PyTorch, irrespective of the host operating system, provided the host has a driver and library installation for CUDA/ROCm.</li>
+<li><strong>Compatility with Docker</strong>: It is 100% OCI compatible and aims for maximum compatibility with Docker.</li>
+<li><strong>Designed for Security</strong>: It allows unprivileged users to use containers while preventing privilege escalation within the container and this means that users are the same within and outside the container.</li>
+<li><strong>Verifiable Reproducibility and Security</strong>: It ensures trustworthy reproducibility and security through cryptographic signatures and an immutable container image format.</li>
+</ul>
+<h2 id="installing-apptainer">Installing Apptainer<a class="headerlink" href="#installing-apptainer" title="Permanent link">&para;</a></h2>
+<p>The following guide assumes a Linux distribution. We have also rigorously tested this on Windows WSL2 Ubuntu which works perfectly, so if you are on a Windows machine, this guide would work too. We are pleasantly surprised till this day, the sheer investment and improvement Windows made to bring WSL2 to a state where it is almost as good as a bare metal Linux distribution installation.</p>
+<h3 id="install-apptainer">Install Apptainer<a class="headerlink" href="#install-apptainer" title="Permanent link">&para;</a></h3>
+<p>All you need is to install <a href="https://apptainer.org/docs/admin/latest/installation.html">Apptainer</a> to be able to leverage on this repository to work in containers with multiple environments (CPU/GPU) with any packages and OS independent of your host (local) machine.</p>
+<h3 id="install-nvidia-libraries">Install NVIDIA libraries<a class="headerlink" href="#install-nvidia-libraries" title="Permanent link">&para;</a></h3>
+<p>I advise you to use the <a href="https://lambdalabs.com/lambda-stack-deep-learning-software">Lambda Stack</a> to manage your drivers effectively without errors. It's a single bash script that allows you to install and upgrade your NVIDIA drivers.</p>
+<p>Check CUDA toolkit installation via <code>nvcc -V</code></p>
+<h3 id="set-nvidia-container-cli-apptainer-paths-deprecated">Set NVIDIA Container CLI Apptainer Paths (Deprecated)<a class="headerlink" href="#set-nvidia-container-cli-apptainer-paths-deprecated" title="Permanent link">&para;</a></h3>
+<p>Not necessary in Apptainer, only in Singularity.</p>
+<p>To get nvidia-container-cli path
+<div class="highlight"><pre><span></span><code>which nvidia-container-cli
+</code></pre></div></p>
+<p>To set
+<div class="highlight"><pre><span></span><code>sudo singularity config global --set &quot;nvidia-container-cli path&quot; &quot;/usr/bin/nvidia-container-cli&quot;
+</code></pre></div></p>
+<p>To check
+<div class="highlight"><pre><span></span><code>sudo singularity config global --get &quot;nvidia-container-cli path&quot;
+</code></pre></div></p>
+<h2 id="how-to-use-apptainer-in-cpugpu-mode">How to Use Apptainer in CPU/GPU Mode<a class="headerlink" href="#how-to-use-apptainer-in-cpugpu-mode" title="Permanent link">&para;</a></h2>
+<h3 id="option-a-transparent-image-container-workflow">Option A: Transparent Image Container Workflow<a class="headerlink" href="#option-a-transparent-image-container-workflow" title="Permanent link">&para;</a></h3>
+<p>This is recommended to build apptainer containers as it's transparent with the greatest reproducibility.</p>
+<h4 id="1-build-container-sif">1. Build container (<code>.sif</code>)<a class="headerlink" href="#1-build-container-sif" title="Permanent link">&para;</a></h4>
+<p>To build and test apptainer image container, simply run the following command in any of the folders:</p>
+<div class="highlight"><pre><span></span><code>bash build_sif.sh
+</code></pre></div>
+<h4 id="2-run-container">2. Run container<a class="headerlink" href="#2-run-container" title="Permanent link">&para;</a></h4>
+<p>For CPU
+<div class="highlight"><pre><span></span><code>apptainer shell apptainer_container_$VER_sandbox.sif
+</code></pre></div></p>
+<p>For GPU using NVIDIA Container CLI
+<div class="highlight"><pre><span></span><code>apptainer shell --nv --nvccli apptainer_container_$VER_sandbox.sif
+</code></pre></div></p>
+<p>A quick way to test if you're able have everything good to run is to run <code>nvidia-smi</code> when you shell into the container. </p>
+<h3 id="notes-on-gpu-availability">Notes on GPU Availability<a class="headerlink" href="#notes-on-gpu-availability" title="Permanent link">&para;</a></h3>
+<p>If you do not have a GPU on your host or local machine, the image will still be built and can be shared and used by machines with GPUs! The test for CUDA will fail on your local machine without GPU which will just spit out an error that you do not have a driver. There's nothing to worry about that.</p>
+<h3 id="option-b-blackbox">Option B: Blackbox<a class="headerlink" href="#option-b-blackbox" title="Permanent link">&para;</a></h3>
+<p>This is not recommended to build production apptainer containers but it is good for experimentation to determine the right configuration to put into your definition file for <code>Option A</code>.</p>
+<p>Note, replace <code>$VER</code> with whatever version is in the specific folder.</p>
+<h4 id="1-build-container-sif_1">1. Build container (<code>.sif</code>)<a class="headerlink" href="#1-build-container-sif_1" title="Permanent link">&para;</a></h4>
+<p>To build and test apptainer image container, simply run the following command in any of the folders:
+<div class="highlight"><pre><span></span><code>bash build_sif.sh
+</code></pre></div></p>
+<h4 id="2-convert-container-to-mutable-sandbox">2. Convert container to mutable sandbox<a class="headerlink" href="#2-convert-container-to-mutable-sandbox" title="Permanent link">&para;</a></h4>
+<div class="highlight"><pre><span></span><code>apptainer build --sandbox apptainer_container_$VER_sandbox apptainer_container_$VER.sif
+</code></pre></div>
+<h4 id="3-shell-into-writable-sandbox">3. Shell into writable sandbox<a class="headerlink" href="#3-shell-into-writable-sandbox" title="Permanent link">&para;</a></h4>
+<p>You can install new packages and make persisting changes to the container in this step for experimentation.
+<div class="highlight"><pre><span></span><code>apptainer shell --writable apptainer_container_$VER_sandbox
+</code></pre></div></p>
+<h4 id="4-convert-container-to-immutable-image-sif">4. Convert container to immutable image (<code>.sif</code>)<a class="headerlink" href="#4-convert-container-to-immutable-image-sif" title="Permanent link">&para;</a></h4>
+<div class="highlight"><pre><span></span><code>apptainer build --sandbox apptainer_container_$VER_sandbox.sif apptainer_container_$VER_img
+</code></pre></div>
+<h2 id="basic-apptainer-commands-cheatsheet">Basic Apptainer Commands Cheatsheet<a class="headerlink" href="#basic-apptainer-commands-cheatsheet" title="Permanent link">&para;</a></h2>
+<p>This provides a list of useful commands.</p>
+<div class="highlight"><pre><span></span><code># test apptainer
+apptainer version
+
+# pull cowsay container from docker
+apptainer pull docker://sylabsio/lolcow
+
+# build (generic that can also build from local definition files, it is like a swiss army knife, more flexible than pull)
+apptainer build lolcow.sif docker://sylabsio/lolcow
+
+# run option 1 requiring to shell into the container
+apptainer shell lolcow.sif
+cowsay moo
+
+# run option 2 without requiring to shell into the container
+apptainer exec lolcow.sif cowsay moo
+
+# clean cache during builds
+apptainer cache clean
+</code></pre></div>
+<h2 id="available-containers-definition-files">Available Containers &amp; Definition Files<a class="headerlink" href="#available-containers-definition-files" title="Permanent link">&para;</a></h2>
+<p>We maintain a repository of Apptainer recipes in this <a href="https://github.com/ritchieng/apptainer-recipes">repository</a>, feel free to clone/download it locally.</p>
+<h3 id="gpu-containers">GPU Containers<a class="headerlink" href="#gpu-containers" title="Permanent link">&para;</a></h3>
+<p>GPU containers can be found in <code>./containers/gpu</code> when you clone the above repository.</p>
+<h4 id="ollama-mistral-7b-workloads">Ollama (Mistral 7b) Workloads<a class="headerlink" href="#ollama-mistral-7b-workloads" title="Permanent link">&para;</a></h4>
+<ul>
+<li>Go into container folder: <code>cd ./containers/gpu/ollama</code></li>
+<li>Run 1<sup>st</sup> session <code>apptainer shell --nv --nvccli apptainer_container_0.1.sif</code><ul>
+<li><code>ollama serve</code></li>
+</ul>
+</li>
+<li>Run 2<sup>nd</sup> session (another window) <code>apptainer shell --nv --nvccli apptainer.1.sif</code><ul>
+<li><code>ollama run mistral</code></li>
+<li>You can now communicate with mistral model in your bash, or any other model you can pull on <a href="https://ollama.com/">ollama website</a></li>
+</ul>
+</li>
+</ul>
+<div class="admonition info">
+<p class="admonition-title">Model Choice</p>
+<p>This runs a Mistral model as an example. You can run any other models by swapping out <code>mistral</code> reference above to any models on <a href="https://ollama.com/library">Ollama's library</a>.</p>
+</div>
+<h4 id="ollama-gemma-workloads">Ollama (Gemma) Workloads<a class="headerlink" href="#ollama-gemma-workloads" title="Permanent link">&para;</a></h4>
+<ul>
+<li>Go into container folder: <code>cd ./containers/gpu/ollama</code></li>
+<li>Run 1<sup>st</sup> session <code>apptainer shell --nv --nvccli apptainer_container_0.1.sif</code><ul>
+<li><code>ollama serve</code></li>
+</ul>
+</li>
+<li>Run 2<sup>nd</sup> session (another window) <code>apptainer shell --nv --nvccli apptainer.1.sif</code><ul>
+<li><code>ollama run gemma:7b</code> or <code>ollama run gemma:2b</code></li>
+<li>You can now communicate with gemma model in your bash, or any other model you can pull on <a href="https://ollama.com/">ollama website</a></li>
+</ul>
+</li>
+</ul>
+<div class="admonition info">
+<p class="admonition-title">Model Choice</p>
+<p>This runs a Gemma 2b or 7b model as an example. You can run any other models by swapping out <code>gemma</code> reference above to any models on <a href="https://ollama.com/library">Ollama's library</a>.</p>
+</div>
+<h4 id="llamaindex-workloads">LLamaindex workloads<a class="headerlink" href="#llamaindex-workloads" title="Permanent link">&para;</a></h4>
+<ul>
+<li>Go into container folder: <code>cd ./containers/gpu/llamaindex</code></li>
+<li>Run 1<sup>st</sup> session (first window)<code>apptainer shell --nv --nvccli apptainer_container_0.1.sif</code><ul>
+<li><code>ollama serve</code></li>
+</ul>
+</li>
+<li>Run 2<sup>nd</sup> session (second window) <code>apptainer shell --nv --nvccli apptainer_container_0.1.sif</code><ul>
+<li><code>ollama run mistral</code></li>
+</ul>
+</li>
+<li>Run 3<sup>rd</sup> session (third window) <code>apptainer shell --nv --nvccli apptainer_container_0.1.sif</code><ul>
+<li><code>python</code></li>
+<li><code>from llama_index.llms import Ollama</code></li>
+<li><code>llm = Ollama(model="mistral")</code></li>
+<li><code>response = llm.complete("What is Singapore")</code></li>
+<li><code>print(response)</code></li>
+</ul>
+</li>
+</ul>
+<div class="admonition info">
+<p class="admonition-title">Model Choice</p>
+<p>This runs a Mistral model as an example. You can run any other models by swapping out <code>mistral</code> reference above to any models on <a href="https://ollama.com/library">Ollama's library</a>.</p>
+</div>
+<h3 id="cpu-containers">CPU Containers<a class="headerlink" href="#cpu-containers" title="Permanent link">&para;</a></h3>
+<p>CPU containers can be found in <code>./containers/cpu</code> when you clone the above repository.</p>
+<h4 id="math-workloads">Math Workloads<a class="headerlink" href="#math-workloads" title="Permanent link">&para;</a></h4>
+<ul>
+<li>Go into container folder: <code>cd ./containers/cpu/math</code><ul>
+<li>Run <code>apptainer shell apptainer_container_0.2.sif</code></li>
+</ul>
+</li>
+</ul>
+<h2 id="summary">Summary<a class="headerlink" href="#summary" title="Permanent link">&para;</a></h2>
+<p>We introduced Apptainer as a way to run CPU or GPU workloads that can scale from your desktops, to on-prem HPC data centers, or to cloud providers like Azure, AWS, and GCP. Subsequently, we introduced running LLMs within Apptainer offline locally with Gemma 2b/7b (Google) and Mistral 7b (Mistral AI) models. With portable container to run our code and models, we can now move to diving deep into LLMs, Multimodal Language Models, and Retrieval Augmented Generation (RAG).</p>
+                
+              
+              
+                
+
+
+  <h2 id="__comments">Comments</h2>
+  <div id="disqus_thread"></div>
+  <script>var disqus_config=function(){this.page.url="https://www.deeplearningwizard.com/language_model/containers/hpc_containers_apptainer/",this.page.identifier="language_model/containers/hpc_containers_apptainer/"};window.addEventListener("load",function(){var e=document,i=e.createElement("script");i.src="//deep-learning-wizard.disqus.com/embed.js",i.setAttribute("data-timestamp",+new Date),(e.head||e.body).appendChild(i)})</script>
+
+              
+            </article>
+          </div>
+        </div>
+        
+          <a href="#" class="md-top md-icon" data-md-component="top" data-md-state="hidden">
+            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12z"/></svg>
+            Back to top
+          </a>
+        
+      </main>
+      
+        
+<footer class="md-footer">
+  
+    <nav class="md-footer__inner md-grid" aria-label="Footer">
+      
+        
+        <a href="../../intro/" class="md-footer__link md-footer__link--prev" aria-label="Previous: Introduction" rel="prev">
+          <div class="md-footer__button md-icon">
+            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12z"/></svg>
+          </div>
+          <div class="md-footer__title">
+            <div class="md-ellipsis">
+              <span class="md-footer__direction">
+                Previous
+              </span>
+              Introduction
+            </div>
+          </div>
+        </a>
+      
+      
+        
+        <a href="../../llm/llm_intro_hyperparameter_tuning/" class="md-footer__link md-footer__link--next" aria-label="Next: LLMs Introduction and Hyperparameter Tuning" rel="next">
+          <div class="md-footer__title">
+            <div class="md-ellipsis">
+              <span class="md-footer__direction">
+                Next
+              </span>
+              LLMs Introduction and Hyperparameter Tuning
+            </div>
+          </div>
+          <div class="md-footer__button md-icon">
+            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11H4z"/></svg>
+          </div>
+        </a>
+      
+    </nav>
+  
+  <div class="md-footer-meta md-typeset">
+    <div class="md-footer-meta__inner md-grid">
+      <div class="md-footer-copyright">
+        
+          <div class="md-footer-copyright__highlight">
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
+          </div>
+        
+        
+      </div>
+      
+  <div class="md-footer-social">
+    
+      
+      
+        
+        
+      
+      <a href="https://www.youtube.com/channel/UCJz2MIjiCosOQCwhnsYxeEw" target="_blank" rel="noopener" title="www.youtube.com" class="md-footer-social__link">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512"><path d="M549.655 124.083c-6.281-23.65-24.787-42.276-48.284-48.597C458.781 64 288 64 288 64S117.22 64 74.629 75.486c-23.497 6.322-42.003 24.947-48.284 48.597-11.412 42.867-11.412 132.305-11.412 132.305s0 89.438 11.412 132.305c6.281 23.65 24.787 41.5 48.284 47.821C117.22 448 288 448 288 448s170.78 0 213.371-11.486c23.497-6.321 42.003-24.171 48.284-47.821 11.412-42.867 11.412-132.305 11.412-132.305s0-89.438-11.412-132.305zm-317.51 213.508V175.185l142.739 81.205-142.739 81.201z"/></svg>
+      </a>
+    
+      
+      
+        
+        
+      
+      <a href="https://twitter.com/deeplearningwiz" target="_blank" rel="noopener" title="twitter.com" class="md-footer-social__link">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M459.37 151.716c.325 4.548.325 9.097.325 13.645 0 138.72-105.583 298.558-298.558 298.558-59.452 0-114.68-17.219-161.137-47.106 8.447.974 16.568 1.299 25.34 1.299 49.055 0 94.213-16.568 130.274-44.832-46.132-.975-84.792-31.188-98.112-72.772 6.498.974 12.995 1.624 19.818 1.624 9.421 0 18.843-1.3 27.614-3.573-48.081-9.747-84.143-51.98-84.143-102.985v-1.299c13.969 7.797 30.214 12.67 47.431 13.319-28.264-18.843-46.781-51.005-46.781-87.391 0-19.492 5.197-37.36 14.294-52.954 51.655 63.675 129.3 105.258 216.365 109.807-1.624-7.797-2.599-15.918-2.599-24.04 0-57.828 46.782-104.934 104.934-104.934 30.213 0 57.502 12.67 76.67 33.137 23.715-4.548 46.456-13.32 66.599-25.34-7.798 24.366-24.366 44.833-46.132 57.827 21.117-2.273 41.584-8.122 60.426-16.243-14.292 20.791-32.161 39.308-52.628 54.253z"/></svg>
+      </a>
+    
+      
+      
+        
+        
+      
+      <a href="https://www.facebook.com/DeepLearningWizard/" target="_blank" rel="noopener" title="www.facebook.com" class="md-footer-social__link">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M504 256C504 119 393 8 256 8S8 119 8 256c0 123.78 90.69 226.38 209.25 245V327.69h-63V256h63v-54.64c0-62.15 37-96.48 93.67-96.48 27.14 0 55.52 4.84 55.52 4.84v61h-31.28c-30.8 0-40.41 19.12-40.41 38.73V256h68.78l-11 71.69h-57.78V501C413.31 482.38 504 379.78 504 256z"/></svg>
+      </a>
+    
+      
+      
+        
+        
+      
+      <a href="https://www.linkedin.com/company/deeplearningwizard/" target="_blank" rel="noopener" title="www.linkedin.com" class="md-footer-social__link">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M416 32H31.9C14.3 32 0 46.5 0 64.3v383.4C0 465.5 14.3 480 31.9 480H416c17.6 0 32-14.5 32-32.3V64.3c0-17.8-14.4-32.3-32-32.3zM135.4 416H69V202.2h66.5V416zm-33.2-243c-21.3 0-38.5-17.3-38.5-38.5S80.9 96 102.2 96c21.2 0 38.5 17.3 38.5 38.5 0 21.3-17.2 38.5-38.5 38.5zm282.1 243h-66.4V312c0-24.8-.5-56.7-34.5-56.7-34.6 0-39.9 27-39.9 54.9V416h-66.4V202.2h63.7v29.2h.9c8.9-16.8 30.6-34.5 62.9-34.5 67.2 0 79.7 44.3 79.7 101.9V416z"/></svg>
+      </a>
+    
+      
+      
+        
+        
+      
+      <a href="https://github.com/ritchieng" target="_blank" rel="noopener" title="github.com" class="md-footer-social__link">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
+      </a>
+    
+  </div>
+
+    </div>
+  </div>
+</footer>
+      
+    </div>
+    <div class="md-dialog" data-md-component="dialog">
+      <div class="md-dialog__inner md-typeset"></div>
+    </div>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
+    
+    
+      <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
+      
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
+      
+    
+  </body>
+</html>
\ No newline at end of file
diff --git a/language_model/intro/index.html b/language_model/intro/index.html
new file mode 100644
index 000000000..48f5cc630
--- /dev/null
+++ b/language_model/intro/index.html
@@ -0,0 +1,1747 @@
+
+<!doctype html>
+<html lang="en" class="no-js">
+  <head>
+    
+      <meta charset="utf-8">
+      <meta name="viewport" content="width=device-width,initial-scale=1">
+      
+        <meta name="description" content="We try to make learning deep learning, deep bayesian learning, and deep reinforcement learning math and code easier. Used by thousands.">
+      
+      
+      
+        <meta name="author" content="Ritchie Ng">
+      
+      
+        <link rel="canonical" href="https://www.deeplearningwizard.com/language_model/intro/">
+      
+      <link rel="icon" href="../../docs/assets/favicon.ico">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
+    
+    
+      
+        <title>Introduction - Deep Learning Wizard</title>
+      
+    
+    
+      <link rel="stylesheet" href="../../assets/stylesheets/main.f7f47774.min.css">
+      
+        
+        <link rel="stylesheet" href="../../assets/stylesheets/palette.3f5d1f46.min.css">
+        
+      
+    
+    
+    
+      
+        
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700%7CRoboto+Mono&display=fallback">
+        <style>:root{--md-text-font-family:"Roboto";--md-code-font-family:"Roboto Mono"}</style>
+      
+    
+    
+    
+    
+      
+
+  
+
+
+  
+
+  
+
+
+  <script>window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)},ga.l=+new Date,ga("create","UA-122083328-1","auto"),ga("set","anonymizeIp",!0),ga("send","pageview"),document.addEventListener("DOMContentLoaded",function(){document.forms.search&&document.forms.search.query.addEventListener("blur",function(){var e;this.value&&(e=document.location.pathname,ga("send","pageview",e+"?q="+this.value))}),"undefined"!=typeof location$&&location$.subscribe(function(e){ga("send","pageview",e.pathname)})})</script>
+  <script async src="https://www.google-analytics.com/analytics.js"></script>
+
+
+    
+    
+  </head>
+  
+  
+    
+    
+      
+    
+    
+    
+    
+    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo">
+  
+    
+    <script>function __prefix(e){return new URL("../..",location).pathname+"."+e}function __get(e,t=localStorage){return JSON.parse(t.getItem(__prefix(e)))}</script>
+    
+      <script>var palette=__get("__palette");if(null!==palette&&"object"==typeof palette.color)for(var key in palette.color)document.body.setAttribute("data-md-color-"+key,palette.color[key])</script>
+    
+    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
+    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
+    <label class="md-overlay" for="__drawer"></label>
+    <div data-md-component="skip">
+      
+        
+        <a href="#language-model" class="md-skip">
+          Skip to content
+        </a>
+      
+    </div>
+    <div data-md-component="announce">
+      
+    </div>
+    
+      <header class="md-header" data-md-component="header">
+  <nav class="md-header__inner md-grid" aria-label="Header">
+    <a href="../.." title="Deep Learning Wizard" class="md-header__button md-logo" aria-label="Deep Learning Wizard" data-md-component="logo">
+      
+  
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M396.8 352h22.4c6.4 0 12.8-6.4 12.8-12.8V108.8c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v230.4c0 6.4 6.4 12.8 12.8 12.8zm-192 0h22.4c6.4 0 12.8-6.4 12.8-12.8V140.8c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v198.4c0 6.4 6.4 12.8 12.8 12.8zm96 0h22.4c6.4 0 12.8-6.4 12.8-12.8V204.8c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v134.4c0 6.4 6.4 12.8 12.8 12.8zM496 400H48V80c0-8.84-7.16-16-16-16H16C7.16 64 0 71.16 0 80v336c0 17.67 14.33 32 32 32h464c8.84 0 16-7.16 16-16v-16c0-8.84-7.16-16-16-16zm-387.2-48h22.4c6.4 0 12.8-6.4 12.8-12.8v-70.4c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v70.4c0 6.4 6.4 12.8 12.8 12.8z"/></svg>
+
+    </a>
+    <label class="md-header__button md-icon" for="__drawer">
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2z"/></svg>
+    </label>
+    <div class="md-header__title" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            Deep Learning Wizard
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+            
+              Introduction
+            
+          </span>
+        </div>
+      </div>
+    </div>
+    
+      <form class="md-header__option" data-md-component="palette">
+        
+          
+          
+          <input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo"  aria-label="Switch to dark mode"  type="radio" name="__palette" id="__palette_1">
+          
+            <label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_2" hidden>
+              <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M7 10a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2m10-3a5 5 0 0 1 5 5 5 5 0 0 1-5 5H7a5 5 0 0 1-5-5 5 5 0 0 1 5-5h10M7 9a3 3 0 0 0-3 3 3 3 0 0 0 3 3h10a3 3 0 0 0 3-3 3 3 0 0 0-3-3H7z"/></svg>
+            </label>
+          
+        
+          
+          
+          <input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="indigo" data-md-color-accent="indigo"  aria-label="Switch to light mode"  type="radio" name="__palette" id="__palette_2">
+          
+            <label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
+              <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 7H7a5 5 0 0 0-5 5 5 5 0 0 0 5 5h10a5 5 0 0 0 5-5 5 5 0 0 0-5-5m0 8a3 3 0 0 1-3-3 3 3 0 0 1 3-3 3 3 0 0 1 3 3 3 3 0 0 1-3 3z"/></svg>
+            </label>
+          
+        
+      </form>
+    
+    
+    
+      <label class="md-header__button md-icon" for="__search">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5z"/></svg>
+      </label>
+      
+<div class="md-search" data-md-component="search" role="dialog">
+  <label class="md-search__overlay" for="__search"></label>
+  <div class="md-search__inner" role="search">
+    <form class="md-search__form" name="search">
+      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
+      <label class="md-search__icon md-icon" for="__search">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5z"/></svg>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12z"/></svg>
+      </label>
+      <nav class="md-search__options" aria-label="Search">
+        
+          <a href="javascript:void(0)" class="md-search__icon md-icon" aria-label="Share" data-clipboard data-clipboard-text="" data-md-component="search-share" tabindex="-1">
+            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M18 16.08c-.76 0-1.44.3-1.96.77L8.91 12.7c.05-.23.09-.46.09-.7 0-.24-.04-.47-.09-.7l7.05-4.11c.54.5 1.25.81 2.04.81a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3c0 .24.04.47.09.7L8.04 9.81C7.5 9.31 6.79 9 6 9a3 3 0 0 0-3 3 3 3 0 0 0 3 3c.79 0 1.5-.31 2.04-.81l7.12 4.15c-.05.21-.08.43-.08.66 0 1.61 1.31 2.91 2.92 2.91 1.61 0 2.92-1.3 2.92-2.91A2.92 2.92 0 0 0 18 16.08z"/></svg>
+          </a>
+        
+        <button type="reset" class="md-search__icon md-icon" aria-label="Clear" tabindex="-1">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41z"/></svg>
+        </button>
+      </nav>
+      
+        <div class="md-search__suggest" data-md-component="search-suggest"></div>
+      
+    </form>
+    <div class="md-search__output">
+      <div class="md-search__scrollwrap" data-md-scrollfix>
+        <div class="md-search-result" data-md-component="search-result">
+          <div class="md-search-result__meta">
+            Initializing search
+          </div>
+          <ol class="md-search-result__list"></ol>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+    
+    
+      <div class="md-header__source">
+        
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+  </div>
+  <div class="md-source__repository">
+    ritchieng/deep-learning-wizard
+  </div>
+</a>
+      </div>
+    
+  </nav>
+</header>
+    
+    <div class="md-container" data-md-component="container">
+      
+      
+        
+          
+<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
+  <div class="md-tabs__inner md-grid">
+    <ul class="md-tabs__list">
+      
+        
+  
+  
+
+
+  <li class="md-tabs__item">
+    <a href="../.." class="md-tabs__link">
+      Home
+    </a>
+  </li>
+
+      
+        
+  
+  
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../home/" class="md-tabs__link">
+        About
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../deep_learning/intro/" class="md-tabs__link">
+        Deep Learning
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+    
+  
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="./" class="md-tabs__link md-tabs__link--active">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../machine_learning/intro/" class="md-tabs__link">
+        Machine Learning
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../programming/intro/" class="md-tabs__link">
+        Programming
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
+    
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../data_engineering/nosql/cassandra/intro/" class="md-tabs__link">
+        Data Engineering
+      </a>
+    </li>
+  
+
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../news/news/" class="md-tabs__link">
+        News
+      </a>
+    </li>
+  
+
+      
+    </ul>
+  </div>
+</nav>
+        
+      
+      <main class="md-main" data-md-component="main">
+        <div class="md-main__inner md-grid">
+          
+            
+              
+              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+                    
+
+  
+
+
+<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0">
+  <label class="md-nav__title" for="__drawer">
+    <a href="../.." title="Deep Learning Wizard" class="md-nav__button md-logo" aria-label="Deep Learning Wizard" data-md-component="logo">
+      
+  
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M396.8 352h22.4c6.4 0 12.8-6.4 12.8-12.8V108.8c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v230.4c0 6.4 6.4 12.8 12.8 12.8zm-192 0h22.4c6.4 0 12.8-6.4 12.8-12.8V140.8c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v198.4c0 6.4 6.4 12.8 12.8 12.8zm96 0h22.4c6.4 0 12.8-6.4 12.8-12.8V204.8c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v134.4c0 6.4 6.4 12.8 12.8 12.8zM496 400H48V80c0-8.84-7.16-16-16-16H16C7.16 64 0 71.16 0 80v336c0 17.67 14.33 32 32 32h464c8.84 0 16-7.16 16-16v-16c0-8.84-7.16-16-16-16zm-387.2-48h22.4c6.4 0 12.8-6.4 12.8-12.8v-70.4c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v70.4c0 6.4 6.4 12.8 12.8 12.8z"/></svg>
+
+    </a>
+    Deep Learning Wizard
+  </label>
+  
+    <div class="md-nav__source">
+      
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+  </div>
+  <div class="md-source__repository">
+    ritchieng/deep-learning-wizard
+  </div>
+</a>
+    </div>
+  
+  <ul class="md-nav__list" data-md-scrollfix>
+    
+      
+      
+      
+
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../.." class="md-nav__link">
+        Home
+      </a>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_2" type="checkbox" id="__nav_2" >
+      
+      <label class="md-nav__link" for="__nav_2">
+        About
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="About" data-md-level="1">
+        <label class="md-nav__title" for="__nav_2">
+          <span class="md-nav__icon md-icon"></span>
+          About
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../home/" class="md-nav__link">
+        Intro
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../about/" class="md-nav__link">
+        About Us
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../review/" class="md-nav__link">
+        Reviews
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../consultancy/" class="md-nav__link">
+        Consultancy
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3" type="checkbox" id="__nav_3" >
+      
+      <label class="md-nav__link" for="__nav_3">
+        Deep Learning
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Deep Learning" data-md-level="1">
+        <label class="md-nav__title" for="__nav_3">
+          <span class="md-nav__icon md-icon"></span>
+          Deep Learning
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/course_progression/" class="md-nav__link">
+        Course Progression
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3_3" type="checkbox" id="__nav_3_3" >
+      
+      <label class="md-nav__link" for="__nav_3_3">
+        Practical Deep Learning with PyTorch
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Practical Deep Learning with PyTorch" data-md-level="2">
+        <label class="md-nav__title" for="__nav_3_3">
+          <span class="md-nav__icon md-icon"></span>
+          Practical Deep Learning with PyTorch
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/practical_pytorch/pytorch_matrices/" class="md-nav__link">
+        Matrices
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/practical_pytorch/pytorch_gradients/" class="md-nav__link">
+        Gradients
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/practical_pytorch/pytorch_linear_regression/" class="md-nav__link">
+        Linear Regression
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/practical_pytorch/pytorch_logistic_regression/" class="md-nav__link">
+        Logistic Regression
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/" class="md-nav__link">
+        Feedforward Neural Networks (FNN)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/" class="md-nav__link">
+        Convolutional Neural Networks (CNN)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/" class="md-nav__link">
+        Recurrent Neural Networks (RNN)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/" class="md-nav__link">
+        Long Short Term Memory Neural Networks (LSTM)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/practical_pytorch/pytorch_autoencoder/" class="md-nav__link">
+        Autoencoders (AE)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/" class="md-nav__link">
+        Fully-connected Overcomplete Autoencoder (AE)
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3_4" type="checkbox" id="__nav_3_4" >
+      
+      <label class="md-nav__link" for="__nav_3_4">
+        Improving Deep Learning with PyTorch
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Improving Deep Learning with PyTorch" data-md-level="2">
+        <label class="md-nav__title" for="__nav_3_4">
+          <span class="md-nav__icon md-icon"></span>
+          Improving Deep Learning with PyTorch
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/" class="md-nav__link">
+        Derivative, Gradient and Jacobian
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/boosting_models_pytorch/forwardpropagation_backpropagation_gradientdescent/" class="md-nav__link">
+        Forward- and Backward-propagation and Gradient Descent (From Scratch FNN Regression)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/boosting_models_pytorch/lr_scheduling/" class="md-nav__link">
+        Learning Rate Scheduling
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/boosting_models_pytorch/optimizers/" class="md-nav__link">
+        Optimization Algorithms
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/" class="md-nav__link">
+        Weight Initialization and Activation Functions
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3_5" type="checkbox" id="__nav_3_5" >
+      
+      <label class="md-nav__link" for="__nav_3_5">
+        Deep Reinforcement Learning with PyTorch
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Deep Reinforcement Learning with PyTorch" data-md-level="2">
+        <label class="md-nav__title" for="__nav_3_5">
+          <span class="md-nav__icon md-icon"></span>
+          Deep Reinforcement Learning with PyTorch
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/deep_reinforcement_learning_pytorch/supervised_to_rl/" class="md-nav__link">
+        Supervised Learning to Reinforcement Learning (RL)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/" class="md-nav__link">
+        Markov Decision Processes (MDP) and Bellman Equations
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/" class="md-nav__link">
+        Dynamic Programming
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3_6" type="checkbox" id="__nav_3_6" >
+      
+      <label class="md-nav__link" for="__nav_3_6">
+        From Scratch Deep Learning with Python/PyTorch
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="From Scratch Deep Learning with Python/PyTorch" data-md-level="2">
+        <label class="md-nav__title" for="__nav_3_6">
+          <span class="md-nav__icon md-icon"></span>
+          From Scratch Deep Learning with Python/PyTorch
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/fromscratch/fromscratch_logistic_regression/" class="md-nav__link">
+        From Scratch Logistic Regression Classification
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3_7" type="checkbox" id="__nav_3_7" >
+      
+      <label class="md-nav__link" for="__nav_3_7">
+        Compute Optimization
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Compute Optimization" data-md-level="2">
+        <label class="md-nav__title" for="__nav_3_7">
+          <span class="md-nav__icon md-icon"></span>
+          Compute Optimization
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../deep_learning/production_pytorch/speed_optimization_basics_numba/" class="md-nav__link">
+        Speed Optimization Basics Numba
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+    
+  
+  
+    
+    <li class="md-nav__item md-nav__item--active md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" checked>
+      
+      <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+    
+  
+  
+    <li class="md-nav__item md-nav__item--active">
+      
+      <input class="md-nav__toggle md-toggle" data-md-toggle="toc" type="checkbox" id="__toc">
+      
+      
+        
+      
+      
+      <a href="./" class="md-nav__link md-nav__link--active">
+        Introduction
+      </a>
+      
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
+        Machine Learning
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
+        <label class="md-nav__title" for="__nav_5">
+          <span class="md-nav__icon md-icon"></span>
+          Machine Learning
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../machine_learning/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
+      
+      <label class="md-nav__link" for="__nav_5_2">
+        RAPIDS cuDF
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
+        <label class="md-nav__title" for="__nav_5_2">
+          <span class="md-nav__icon md-icon"></span>
+          RAPIDS cuDF
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../machine_learning/gpu/rapids_cudf/" class="md-nav__link">
+        GPU DataFrames
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../machine_learning/gpu/gpu_fractional_differencing/" class="md-nav__link">
+        GPU/CPU Fractional Differencing
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+      
+      <label class="md-nav__link" for="__nav_6">
+        Programming
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Programming" data-md-level="1">
+        <label class="md-nav__title" for="__nav_6">
+          <span class="md-nav__icon md-icon"></span>
+          Programming
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../programming/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../programming/cpp/cpp/" class="md-nav__link">
+        C++
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../programming/bash/bash/" class="md-nav__link">
+        Bash
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../programming/python/python/" class="md-nav__link">
+        Python
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../programming/r/r/" class="md-nav__link">
+        R
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../programming/javascript/javascript/" class="md-nav__link">
+        Javascript
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../programming/electron/electron/" class="md-nav__link">
+        Electron
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../programming/sympy/calculus_sympy/" class="md-nav__link">
+        Sympy
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../programming/numpycupy/linalg_numpy_cupy/" class="md-nav__link">
+        NumPy and CuPy
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+      
+      <label class="md-nav__link" for="__nav_7">
+        Data Engineering
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
+        <label class="md-nav__title" for="__nav_7">
+          <span class="md-nav__icon md-icon"></span>
+          Data Engineering
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
+      
+      <label class="md-nav__link" for="__nav_7_1">
+        Cassandra (NoSQL)
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
+        <label class="md-nav__title" for="__nav_7_1">
+          <span class="md-nav__icon md-icon"></span>
+          Cassandra (NoSQL)
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../data_engineering/nosql/cassandra/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../data_engineering/nosql/cassandra/setting_up_cluster/" class="md-nav__link">
+        Cassandra Cluster Setup
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
+      
+      <label class="md-nav__link" for="__nav_8">
+        News
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="News" data-md-level="1">
+        <label class="md-nav__title" for="__nav_8">
+          <span class="md-nav__icon md-icon"></span>
+          News
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../news/news/" class="md-nav__link">
+        Welcome
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../news/dbs_gpu_rapids_nvidia_ensemble_frac_diff/" class="md-nav__link">
+        Fractional Differencing with GPU (GFD), DBS and NVIDIA, September 2019
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../news/defence_and_science_technology_agency_dsta_nvidia_talk_2016_06/" class="md-nav__link">
+        Deep Learning Introduction, Defence and Science Technology Agency (DSTA) and NVIDIA, June 2019
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../news/detect_waterbone_debris_ai_for_social_good_icml_2019_06/" class="md-nav__link">
+        Oral Presentation for AI for Social Good Workshop ICML, June 2019
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../news/it_youth_leader_2019_03_11/" class="md-nav__link">
+        IT Youth Leader of The Year 2019, March 2019
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../news/ammi_facebook_google_recap_2018_11_21/" class="md-nav__link">
+        AMMI (AIMS) supported by Facebook and Google, November 2018
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../news/nanjing_next_nus_tsinghua_ai_finance_healthcare_2018_11_01/" class="md-nav__link">
+        NExT++ AI in Healthcare and Finance, Nanjing, November 2018
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../news/facebook_pytorch_devcon_recap_2018_10_02/" class="md-nav__link">
+        Recap of Facebook PyTorch Developer Conference, San Francisco, September 2018
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../news/facebook_pytorch_developer_conference_2018_09_05/" class="md-nav__link">
+        Facebook PyTorch Developer Conference, San Francisco, September 2018
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../news/nvidia_nus_mit_datathon_2018_07_05/" class="md-nav__link">
+        NUS-MIT-NUHS NVIDIA Image Recognition Workshop, Singapore, July 2018
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../news/deep_learning_wizard_1y_2018_06_01/" class="md-nav__link">
+        Featured on PyTorch Website 2018
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../news/nvidia_self_driving_cars_talk_2017_06_21/" class="md-nav__link">
+        NVIDIA Self Driving Cars & Healthcare Talk, Singapore, June 2017
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../news/deep_learning_wizard_nvidia_inception_2018_05_01/" class="md-nav__link">
+        NVIDIA Inception Partner Status, Singapore, May 2017
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+  </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+            
+            
+              
+              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+                    
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+  
+  
+  
+    
+  
+  
+</nav>
+                  </div>
+                </div>
+              </div>
+            
+          
+          <div class="md-content" data-md-component="content">
+            <article class="md-content__inner md-typeset">
+              
+                
+                  <a href="https://github.com/ritchieng/deep-learning-wizard/edit/master/docs/language_model/intro.md" title="Edit this page" class="md-content__button md-icon">
+                    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20.71 7.04c.39-.39.39-1.04 0-1.41l-2.34-2.34c-.37-.39-1.02-.39-1.41 0l-1.84 1.83 3.75 3.75M3 17.25V21h3.75L17.81 9.93l-3.75-3.75L3 17.25z"/></svg>
+                  </a>
+                
+                
+                <h1 id="language-model">Language Model<a class="headerlink" href="#language-model" title="Permanent link">&para;</a></h1>
+<p>We'll be covering both CPU and GPU implementations of Large Language Models (LLMs) and Multimodal Language Models.</p>
+<div class="tabbed-set" data-tabs="1:1"><input checked="checked" id="language-model-packages-and-languages-you-will-learn-to-use" name="__tabbed_1" type="radio" /><label for="language-model-packages-and-languages-you-will-learn-to-use">Packages and Languages you will Learn to Use</label><div class="tabbed-content">
+<ul>
+<li>Python</li>
+<li>PyTorch</li>
+<li>Llamaindex</li>
+<li>Ollama</li>
+<li>Huggingface</li>
+<li>Apptainer</li>
+</ul>
+</div>
+</div>
+                
+                  
+                
+              
+              
+                
+
+  
+
+
+              
+            </article>
+          </div>
+        </div>
+        
+          <a href="#" class="md-top md-icon" data-md-component="top" data-md-state="hidden">
+            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12z"/></svg>
+            Back to top
+          </a>
+        
+      </main>
+      
+        
+<footer class="md-footer">
+  
+    <nav class="md-footer__inner md-grid" aria-label="Footer">
+      
+        
+        <a href="../../deep_learning/production_pytorch/speed_optimization_basics_numba/" class="md-footer__link md-footer__link--prev" aria-label="Previous: Speed Optimization Basics Numba" rel="prev">
+          <div class="md-footer__button md-icon">
+            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12z"/></svg>
+          </div>
+          <div class="md-footer__title">
+            <div class="md-ellipsis">
+              <span class="md-footer__direction">
+                Previous
+              </span>
+              Speed Optimization Basics Numba
+            </div>
+          </div>
+        </a>
+      
+      
+        
+        <a href="../containers/hpc_containers_apptainer/" class="md-footer__link md-footer__link--next" aria-label="Next: HPC Containers with Apptainer" rel="next">
+          <div class="md-footer__title">
+            <div class="md-ellipsis">
+              <span class="md-footer__direction">
+                Next
+              </span>
+              HPC Containers with Apptainer
+            </div>
+          </div>
+          <div class="md-footer__button md-icon">
+            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11H4z"/></svg>
+          </div>
+        </a>
+      
+    </nav>
+  
+  <div class="md-footer-meta md-typeset">
+    <div class="md-footer-meta__inner md-grid">
+      <div class="md-footer-copyright">
+        
+          <div class="md-footer-copyright__highlight">
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
+          </div>
+        
+        
+      </div>
+      
+  <div class="md-footer-social">
+    
+      
+      
+        
+        
+      
+      <a href="https://www.youtube.com/channel/UCJz2MIjiCosOQCwhnsYxeEw" target="_blank" rel="noopener" title="www.youtube.com" class="md-footer-social__link">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512"><path d="M549.655 124.083c-6.281-23.65-24.787-42.276-48.284-48.597C458.781 64 288 64 288 64S117.22 64 74.629 75.486c-23.497 6.322-42.003 24.947-48.284 48.597-11.412 42.867-11.412 132.305-11.412 132.305s0 89.438 11.412 132.305c6.281 23.65 24.787 41.5 48.284 47.821C117.22 448 288 448 288 448s170.78 0 213.371-11.486c23.497-6.321 42.003-24.171 48.284-47.821 11.412-42.867 11.412-132.305 11.412-132.305s0-89.438-11.412-132.305zm-317.51 213.508V175.185l142.739 81.205-142.739 81.201z"/></svg>
+      </a>
+    
+      
+      
+        
+        
+      
+      <a href="https://twitter.com/deeplearningwiz" target="_blank" rel="noopener" title="twitter.com" class="md-footer-social__link">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M459.37 151.716c.325 4.548.325 9.097.325 13.645 0 138.72-105.583 298.558-298.558 298.558-59.452 0-114.68-17.219-161.137-47.106 8.447.974 16.568 1.299 25.34 1.299 49.055 0 94.213-16.568 130.274-44.832-46.132-.975-84.792-31.188-98.112-72.772 6.498.974 12.995 1.624 19.818 1.624 9.421 0 18.843-1.3 27.614-3.573-48.081-9.747-84.143-51.98-84.143-102.985v-1.299c13.969 7.797 30.214 12.67 47.431 13.319-28.264-18.843-46.781-51.005-46.781-87.391 0-19.492 5.197-37.36 14.294-52.954 51.655 63.675 129.3 105.258 216.365 109.807-1.624-7.797-2.599-15.918-2.599-24.04 0-57.828 46.782-104.934 104.934-104.934 30.213 0 57.502 12.67 76.67 33.137 23.715-4.548 46.456-13.32 66.599-25.34-7.798 24.366-24.366 44.833-46.132 57.827 21.117-2.273 41.584-8.122 60.426-16.243-14.292 20.791-32.161 39.308-52.628 54.253z"/></svg>
+      </a>
+    
+      
+      
+        
+        
+      
+      <a href="https://www.facebook.com/DeepLearningWizard/" target="_blank" rel="noopener" title="www.facebook.com" class="md-footer-social__link">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M504 256C504 119 393 8 256 8S8 119 8 256c0 123.78 90.69 226.38 209.25 245V327.69h-63V256h63v-54.64c0-62.15 37-96.48 93.67-96.48 27.14 0 55.52 4.84 55.52 4.84v61h-31.28c-30.8 0-40.41 19.12-40.41 38.73V256h68.78l-11 71.69h-57.78V501C413.31 482.38 504 379.78 504 256z"/></svg>
+      </a>
+    
+      
+      
+        
+        
+      
+      <a href="https://www.linkedin.com/company/deeplearningwizard/" target="_blank" rel="noopener" title="www.linkedin.com" class="md-footer-social__link">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M416 32H31.9C14.3 32 0 46.5 0 64.3v383.4C0 465.5 14.3 480 31.9 480H416c17.6 0 32-14.5 32-32.3V64.3c0-17.8-14.4-32.3-32-32.3zM135.4 416H69V202.2h66.5V416zm-33.2-243c-21.3 0-38.5-17.3-38.5-38.5S80.9 96 102.2 96c21.2 0 38.5 17.3 38.5 38.5 0 21.3-17.2 38.5-38.5 38.5zm282.1 243h-66.4V312c0-24.8-.5-56.7-34.5-56.7-34.6 0-39.9 27-39.9 54.9V416h-66.4V202.2h63.7v29.2h.9c8.9-16.8 30.6-34.5 62.9-34.5 67.2 0 79.7 44.3 79.7 101.9V416z"/></svg>
+      </a>
+    
+      
+      
+        
+        
+      
+      <a href="https://github.com/ritchieng" target="_blank" rel="noopener" title="github.com" class="md-footer-social__link">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
+      </a>
+    
+  </div>
+
+    </div>
+  </div>
+</footer>
+      
+    </div>
+    <div class="md-dialog" data-md-component="dialog">
+      <div class="md-dialog__inner md-typeset"></div>
+    </div>
+    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
+    
+    
+      <script src="../../assets/javascripts/bundle.56838a2c.min.js"></script>
+      
+        <script src="../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
+      
+    
+  </body>
+</html>
\ No newline at end of file
diff --git a/language_model/llm/llm_intro_hyperparameter_tuning/index.html b/language_model/llm/llm_intro_hyperparameter_tuning/index.html
new file mode 100644
index 000000000..9f82cc251
--- /dev/null
+++ b/language_model/llm/llm_intro_hyperparameter_tuning/index.html
@@ -0,0 +1,2455 @@
+
+<!doctype html>
+<html lang="en" class="no-js">
+  <head>
+    
+      <meta charset="utf-8">
+      <meta name="viewport" content="width=device-width,initial-scale=1">
+      
+        <meta name="description" content="We try to make learning deep learning, deep bayesian learning, and deep reinforcement learning math and code easier. Used by thousands.">
+      
+      
+      
+        <meta name="author" content="Ritchie Ng">
+      
+      
+        <link rel="canonical" href="https://www.deeplearningwizard.com/language_model/llm/llm_intro_hyperparameter_tuning/">
+      
+      <link rel="icon" href="../../../docs/assets/favicon.ico">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
+    
+    
+      
+        <title>LLMs Introduction and Hyperparameter Tuning - Deep Learning Wizard</title>
+      
+    
+    
+      <link rel="stylesheet" href="../../../assets/stylesheets/main.f7f47774.min.css">
+      
+        
+        <link rel="stylesheet" href="../../../assets/stylesheets/palette.3f5d1f46.min.css">
+        
+      
+    
+    
+    
+      
+        
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700%7CRoboto+Mono&display=fallback">
+        <style>:root{--md-text-font-family:"Roboto";--md-code-font-family:"Roboto Mono"}</style>
+      
+    
+    
+    
+    
+      
+
+  
+
+
+  
+
+  
+
+
+  <script>window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)},ga.l=+new Date,ga("create","UA-122083328-1","auto"),ga("set","anonymizeIp",!0),ga("send","pageview"),document.addEventListener("DOMContentLoaded",function(){document.forms.search&&document.forms.search.query.addEventListener("blur",function(){var e;this.value&&(e=document.location.pathname,ga("send","pageview",e+"?q="+this.value))}),"undefined"!=typeof location$&&location$.subscribe(function(e){ga("send","pageview",e.pathname)})})</script>
+  <script async src="https://www.google-analytics.com/analytics.js"></script>
+
+
+    
+    
+  </head>
+  
+  
+    
+    
+      
+    
+    
+    
+    
+    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo">
+  
+    
+    <script>function __prefix(e){return new URL("../../..",location).pathname+"."+e}function __get(e,t=localStorage){return JSON.parse(t.getItem(__prefix(e)))}</script>
+    
+      <script>var palette=__get("__palette");if(null!==palette&&"object"==typeof palette.color)for(var key in palette.color)document.body.setAttribute("data-md-color-"+key,palette.color[key])</script>
+    
+    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
+    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
+    <label class="md-overlay" for="__drawer"></label>
+    <div data-md-component="skip">
+      
+        
+        <a href="#llm-introduction-hyperparameter-tuning" class="md-skip">
+          Skip to content
+        </a>
+      
+    </div>
+    <div data-md-component="announce">
+      
+    </div>
+    
+      <header class="md-header" data-md-component="header">
+  <nav class="md-header__inner md-grid" aria-label="Header">
+    <a href="../../.." title="Deep Learning Wizard" class="md-header__button md-logo" aria-label="Deep Learning Wizard" data-md-component="logo">
+      
+  
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M396.8 352h22.4c6.4 0 12.8-6.4 12.8-12.8V108.8c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v230.4c0 6.4 6.4 12.8 12.8 12.8zm-192 0h22.4c6.4 0 12.8-6.4 12.8-12.8V140.8c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v198.4c0 6.4 6.4 12.8 12.8 12.8zm96 0h22.4c6.4 0 12.8-6.4 12.8-12.8V204.8c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v134.4c0 6.4 6.4 12.8 12.8 12.8zM496 400H48V80c0-8.84-7.16-16-16-16H16C7.16 64 0 71.16 0 80v336c0 17.67 14.33 32 32 32h464c8.84 0 16-7.16 16-16v-16c0-8.84-7.16-16-16-16zm-387.2-48h22.4c6.4 0 12.8-6.4 12.8-12.8v-70.4c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v70.4c0 6.4 6.4 12.8 12.8 12.8z"/></svg>
+
+    </a>
+    <label class="md-header__button md-icon" for="__drawer">
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2z"/></svg>
+    </label>
+    <div class="md-header__title" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            Deep Learning Wizard
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+            
+              LLMs Introduction and Hyperparameter Tuning
+            
+          </span>
+        </div>
+      </div>
+    </div>
+    
+      <form class="md-header__option" data-md-component="palette">
+        
+          
+          
+          <input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo"  aria-label="Switch to dark mode"  type="radio" name="__palette" id="__palette_1">
+          
+            <label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_2" hidden>
+              <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M7 10a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2m10-3a5 5 0 0 1 5 5 5 5 0 0 1-5 5H7a5 5 0 0 1-5-5 5 5 0 0 1 5-5h10M7 9a3 3 0 0 0-3 3 3 3 0 0 0 3 3h10a3 3 0 0 0 3-3 3 3 0 0 0-3-3H7z"/></svg>
+            </label>
+          
+        
+          
+          
+          <input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="indigo" data-md-color-accent="indigo"  aria-label="Switch to light mode"  type="radio" name="__palette" id="__palette_2">
+          
+            <label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
+              <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 7H7a5 5 0 0 0-5 5 5 5 0 0 0 5 5h10a5 5 0 0 0 5-5 5 5 0 0 0-5-5m0 8a3 3 0 0 1-3-3 3 3 0 0 1 3-3 3 3 0 0 1 3 3 3 3 0 0 1-3 3z"/></svg>
+            </label>
+          
+        
+      </form>
+    
+    
+    
+      <label class="md-header__button md-icon" for="__search">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5z"/></svg>
+      </label>
+      
+<div class="md-search" data-md-component="search" role="dialog">
+  <label class="md-search__overlay" for="__search"></label>
+  <div class="md-search__inner" role="search">
+    <form class="md-search__form" name="search">
+      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
+      <label class="md-search__icon md-icon" for="__search">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5z"/></svg>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12z"/></svg>
+      </label>
+      <nav class="md-search__options" aria-label="Search">
+        
+          <a href="javascript:void(0)" class="md-search__icon md-icon" aria-label="Share" data-clipboard data-clipboard-text="" data-md-component="search-share" tabindex="-1">
+            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M18 16.08c-.76 0-1.44.3-1.96.77L8.91 12.7c.05-.23.09-.46.09-.7 0-.24-.04-.47-.09-.7l7.05-4.11c.54.5 1.25.81 2.04.81a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3c0 .24.04.47.09.7L8.04 9.81C7.5 9.31 6.79 9 6 9a3 3 0 0 0-3 3 3 3 0 0 0 3 3c.79 0 1.5-.31 2.04-.81l7.12 4.15c-.05.21-.08.43-.08.66 0 1.61 1.31 2.91 2.92 2.91 1.61 0 2.92-1.3 2.92-2.91A2.92 2.92 0 0 0 18 16.08z"/></svg>
+          </a>
+        
+        <button type="reset" class="md-search__icon md-icon" aria-label="Clear" tabindex="-1">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41z"/></svg>
+        </button>
+      </nav>
+      
+        <div class="md-search__suggest" data-md-component="search-suggest"></div>
+      
+    </form>
+    <div class="md-search__output">
+      <div class="md-search__scrollwrap" data-md-scrollfix>
+        <div class="md-search-result" data-md-component="search-result">
+          <div class="md-search-result__meta">
+            Initializing search
+          </div>
+          <ol class="md-search-result__list"></ol>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+    
+    
+      <div class="md-header__source">
+        
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+  </div>
+  <div class="md-source__repository">
+    ritchieng/deep-learning-wizard
+  </div>
+</a>
+      </div>
+    
+  </nav>
+</header>
+    
+    <div class="md-container" data-md-component="container">
+      
+      
+        
+          
+<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
+  <div class="md-tabs__inner md-grid">
+    <ul class="md-tabs__list">
+      
+        
+  
+  
+
+
+  <li class="md-tabs__item">
+    <a href="../../.." class="md-tabs__link">
+      Home
+    </a>
+  </li>
+
+      
+        
+  
+  
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../home/" class="md-tabs__link">
+        About
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../deep_learning/intro/" class="md-tabs__link">
+        Deep Learning
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+    
+  
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../intro/" class="md-tabs__link md-tabs__link--active">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../machine_learning/intro/" class="md-tabs__link">
+        Machine Learning
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../programming/intro/" class="md-tabs__link">
+        Programming
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
+    
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../data_engineering/nosql/cassandra/intro/" class="md-tabs__link">
+        Data Engineering
+      </a>
+    </li>
+  
+
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../news/news/" class="md-tabs__link">
+        News
+      </a>
+    </li>
+  
+
+      
+    </ul>
+  </div>
+</nav>
+        
+      
+      <main class="md-main" data-md-component="main">
+        <div class="md-main__inner md-grid">
+          
+            
+              
+              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+                    
+
+  
+
+
+<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0">
+  <label class="md-nav__title" for="__drawer">
+    <a href="../../.." title="Deep Learning Wizard" class="md-nav__button md-logo" aria-label="Deep Learning Wizard" data-md-component="logo">
+      
+  
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M396.8 352h22.4c6.4 0 12.8-6.4 12.8-12.8V108.8c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v230.4c0 6.4 6.4 12.8 12.8 12.8zm-192 0h22.4c6.4 0 12.8-6.4 12.8-12.8V140.8c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v198.4c0 6.4 6.4 12.8 12.8 12.8zm96 0h22.4c6.4 0 12.8-6.4 12.8-12.8V204.8c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v134.4c0 6.4 6.4 12.8 12.8 12.8zM496 400H48V80c0-8.84-7.16-16-16-16H16C7.16 64 0 71.16 0 80v336c0 17.67 14.33 32 32 32h464c8.84 0 16-7.16 16-16v-16c0-8.84-7.16-16-16-16zm-387.2-48h22.4c6.4 0 12.8-6.4 12.8-12.8v-70.4c0-6.4-6.4-12.8-12.8-12.8h-22.4c-6.4 0-12.8 6.4-12.8 12.8v70.4c0 6.4 6.4 12.8 12.8 12.8z"/></svg>
+
+    </a>
+    Deep Learning Wizard
+  </label>
+  
+    <div class="md-nav__source">
+      
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+  </div>
+  <div class="md-source__repository">
+    ritchieng/deep-learning-wizard
+  </div>
+</a>
+    </div>
+  
+  <ul class="md-nav__list" data-md-scrollfix>
+    
+      
+      
+      
+
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../.." class="md-nav__link">
+        Home
+      </a>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_2" type="checkbox" id="__nav_2" >
+      
+      <label class="md-nav__link" for="__nav_2">
+        About
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="About" data-md-level="1">
+        <label class="md-nav__title" for="__nav_2">
+          <span class="md-nav__icon md-icon"></span>
+          About
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../home/" class="md-nav__link">
+        Intro
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../about/" class="md-nav__link">
+        About Us
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../review/" class="md-nav__link">
+        Reviews
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../consultancy/" class="md-nav__link">
+        Consultancy
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3" type="checkbox" id="__nav_3" >
+      
+      <label class="md-nav__link" for="__nav_3">
+        Deep Learning
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Deep Learning" data-md-level="1">
+        <label class="md-nav__title" for="__nav_3">
+          <span class="md-nav__icon md-icon"></span>
+          Deep Learning
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/course_progression/" class="md-nav__link">
+        Course Progression
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3_3" type="checkbox" id="__nav_3_3" >
+      
+      <label class="md-nav__link" for="__nav_3_3">
+        Practical Deep Learning with PyTorch
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Practical Deep Learning with PyTorch" data-md-level="2">
+        <label class="md-nav__title" for="__nav_3_3">
+          <span class="md-nav__icon md-icon"></span>
+          Practical Deep Learning with PyTorch
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/practical_pytorch/pytorch_matrices/" class="md-nav__link">
+        Matrices
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/practical_pytorch/pytorch_gradients/" class="md-nav__link">
+        Gradients
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/practical_pytorch/pytorch_linear_regression/" class="md-nav__link">
+        Linear Regression
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/practical_pytorch/pytorch_logistic_regression/" class="md-nav__link">
+        Logistic Regression
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/" class="md-nav__link">
+        Feedforward Neural Networks (FNN)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/" class="md-nav__link">
+        Convolutional Neural Networks (CNN)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/" class="md-nav__link">
+        Recurrent Neural Networks (RNN)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/" class="md-nav__link">
+        Long Short Term Memory Neural Networks (LSTM)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/practical_pytorch/pytorch_autoencoder/" class="md-nav__link">
+        Autoencoders (AE)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/" class="md-nav__link">
+        Fully-connected Overcomplete Autoencoder (AE)
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3_4" type="checkbox" id="__nav_3_4" >
+      
+      <label class="md-nav__link" for="__nav_3_4">
+        Improving Deep Learning with PyTorch
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Improving Deep Learning with PyTorch" data-md-level="2">
+        <label class="md-nav__title" for="__nav_3_4">
+          <span class="md-nav__icon md-icon"></span>
+          Improving Deep Learning with PyTorch
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/" class="md-nav__link">
+        Derivative, Gradient and Jacobian
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/boosting_models_pytorch/forwardpropagation_backpropagation_gradientdescent/" class="md-nav__link">
+        Forward- and Backward-propagation and Gradient Descent (From Scratch FNN Regression)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/boosting_models_pytorch/lr_scheduling/" class="md-nav__link">
+        Learning Rate Scheduling
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/boosting_models_pytorch/optimizers/" class="md-nav__link">
+        Optimization Algorithms
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/" class="md-nav__link">
+        Weight Initialization and Activation Functions
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3_5" type="checkbox" id="__nav_3_5" >
+      
+      <label class="md-nav__link" for="__nav_3_5">
+        Deep Reinforcement Learning with PyTorch
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Deep Reinforcement Learning with PyTorch" data-md-level="2">
+        <label class="md-nav__title" for="__nav_3_5">
+          <span class="md-nav__icon md-icon"></span>
+          Deep Reinforcement Learning with PyTorch
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/deep_reinforcement_learning_pytorch/supervised_to_rl/" class="md-nav__link">
+        Supervised Learning to Reinforcement Learning (RL)
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/" class="md-nav__link">
+        Markov Decision Processes (MDP) and Bellman Equations
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/" class="md-nav__link">
+        Dynamic Programming
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3_6" type="checkbox" id="__nav_3_6" >
+      
+      <label class="md-nav__link" for="__nav_3_6">
+        From Scratch Deep Learning with Python/PyTorch
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="From Scratch Deep Learning with Python/PyTorch" data-md-level="2">
+        <label class="md-nav__title" for="__nav_3_6">
+          <span class="md-nav__icon md-icon"></span>
+          From Scratch Deep Learning with Python/PyTorch
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/fromscratch/fromscratch_logistic_regression/" class="md-nav__link">
+        From Scratch Logistic Regression Classification
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3_7" type="checkbox" id="__nav_3_7" >
+      
+      <label class="md-nav__link" for="__nav_3_7">
+        Compute Optimization
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Compute Optimization" data-md-level="2">
+        <label class="md-nav__title" for="__nav_3_7">
+          <span class="md-nav__icon md-icon"></span>
+          Compute Optimization
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../deep_learning/production_pytorch/speed_optimization_basics_numba/" class="md-nav__link">
+        Speed Optimization Basics Numba
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+    
+  
+  
+    
+    <li class="md-nav__item md-nav__item--active md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" checked>
+      
+      <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+    
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" checked>
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+    
+  
+  
+    <li class="md-nav__item md-nav__item--active">
+      
+      <input class="md-nav__toggle md-toggle" data-md-toggle="toc" type="checkbox" id="__toc">
+      
+      
+        
+      
+      
+        <label class="md-nav__link md-nav__link--active" for="__toc">
+          LLMs Introduction and Hyperparameter Tuning
+          <span class="md-nav__icon md-icon"></span>
+        </label>
+      
+      <a href="./" class="md-nav__link md-nav__link--active">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+      
+        
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+  
+  
+  
+    
+  
+  
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+      
+        <li class="md-nav__item">
+  <a href="#environment-setup" class="md-nav__link">
+    Environment Setup
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#question-and-answer-no-hyperparameter-tuning" class="md-nav__link">
+    Question and Answer | No Hyperparameter Tuning
+  </a>
+  
+    <nav class="md-nav" aria-label="Question and Answer | No Hyperparameter Tuning">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#question-1" class="md-nav__link">
+    Question 1
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#question-2" class="md-nav__link">
+    Question 2
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#question-and-answer-hyperparameter-tuning" class="md-nav__link">
+    Question and Answer | Hyperparameter Tuning
+  </a>
+  
+    <nav class="md-nav" aria-label="Question and Answer | Hyperparameter Tuning">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#temperature-tuning" class="md-nav__link">
+    Temperature Tuning
+  </a>
+  
+    <nav class="md-nav" aria-label="Temperature Tuning">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#low-temperature" class="md-nav__link">
+    Low Temperature
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#high-temperature" class="md-nav__link">
+    High Temperature
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#mathematical-interpretation" class="md-nav__link">
+    Mathematical Interpretation
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#top-k-tuning" class="md-nav__link">
+    Top-K Tuning
+  </a>
+  
+    <nav class="md-nav" aria-label="Top-K Tuning">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#low-k" class="md-nav__link">
+    Low K
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#high-k" class="md-nav__link">
+    High K
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#mathematical-interpretation_1" class="md-nav__link">
+    Mathematical Interpretation
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#summary" class="md-nav__link">
+    Summary
+  </a>
+  
+</li>
+      
+    </ul>
+  
+</nav>
+      
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
+        Machine Learning
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
+        <label class="md-nav__title" for="__nav_5">
+          <span class="md-nav__icon md-icon"></span>
+          Machine Learning
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../machine_learning/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
+      
+      <label class="md-nav__link" for="__nav_5_2">
+        RAPIDS cuDF
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
+        <label class="md-nav__title" for="__nav_5_2">
+          <span class="md-nav__icon md-icon"></span>
+          RAPIDS cuDF
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../machine_learning/gpu/rapids_cudf/" class="md-nav__link">
+        GPU DataFrames
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../machine_learning/gpu/gpu_fractional_differencing/" class="md-nav__link">
+        GPU/CPU Fractional Differencing
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+      
+      <label class="md-nav__link" for="__nav_6">
+        Programming
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Programming" data-md-level="1">
+        <label class="md-nav__title" for="__nav_6">
+          <span class="md-nav__icon md-icon"></span>
+          Programming
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../programming/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../programming/cpp/cpp/" class="md-nav__link">
+        C++
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../programming/bash/bash/" class="md-nav__link">
+        Bash
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../programming/python/python/" class="md-nav__link">
+        Python
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../programming/r/r/" class="md-nav__link">
+        R
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../programming/javascript/javascript/" class="md-nav__link">
+        Javascript
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../programming/electron/electron/" class="md-nav__link">
+        Electron
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../programming/sympy/calculus_sympy/" class="md-nav__link">
+        Sympy
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../programming/numpycupy/linalg_numpy_cupy/" class="md-nav__link">
+        NumPy and CuPy
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+      
+      <label class="md-nav__link" for="__nav_7">
+        Data Engineering
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
+        <label class="md-nav__title" for="__nav_7">
+          <span class="md-nav__icon md-icon"></span>
+          Data Engineering
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
+      
+      <label class="md-nav__link" for="__nav_7_1">
+        Cassandra (NoSQL)
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
+        <label class="md-nav__title" for="__nav_7_1">
+          <span class="md-nav__icon md-icon"></span>
+          Cassandra (NoSQL)
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../data_engineering/nosql/cassandra/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../data_engineering/nosql/cassandra/setting_up_cluster/" class="md-nav__link">
+        Cassandra Cluster Setup
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
+      
+      <label class="md-nav__link" for="__nav_8">
+        News
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="News" data-md-level="1">
+        <label class="md-nav__title" for="__nav_8">
+          <span class="md-nav__icon md-icon"></span>
+          News
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/news/" class="md-nav__link">
+        Welcome
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/dbs_gpu_rapids_nvidia_ensemble_frac_diff/" class="md-nav__link">
+        Fractional Differencing with GPU (GFD), DBS and NVIDIA, September 2019
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/defence_and_science_technology_agency_dsta_nvidia_talk_2016_06/" class="md-nav__link">
+        Deep Learning Introduction, Defence and Science Technology Agency (DSTA) and NVIDIA, June 2019
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/detect_waterbone_debris_ai_for_social_good_icml_2019_06/" class="md-nav__link">
+        Oral Presentation for AI for Social Good Workshop ICML, June 2019
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/it_youth_leader_2019_03_11/" class="md-nav__link">
+        IT Youth Leader of The Year 2019, March 2019
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/ammi_facebook_google_recap_2018_11_21/" class="md-nav__link">
+        AMMI (AIMS) supported by Facebook and Google, November 2018
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/nanjing_next_nus_tsinghua_ai_finance_healthcare_2018_11_01/" class="md-nav__link">
+        NExT++ AI in Healthcare and Finance, Nanjing, November 2018
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/facebook_pytorch_devcon_recap_2018_10_02/" class="md-nav__link">
+        Recap of Facebook PyTorch Developer Conference, San Francisco, September 2018
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/facebook_pytorch_developer_conference_2018_09_05/" class="md-nav__link">
+        Facebook PyTorch Developer Conference, San Francisco, September 2018
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/nvidia_nus_mit_datathon_2018_07_05/" class="md-nav__link">
+        NUS-MIT-NUHS NVIDIA Image Recognition Workshop, Singapore, July 2018
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/deep_learning_wizard_1y_2018_06_01/" class="md-nav__link">
+        Featured on PyTorch Website 2018
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/nvidia_self_driving_cars_talk_2017_06_21/" class="md-nav__link">
+        NVIDIA Self Driving Cars & Healthcare Talk, Singapore, June 2017
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../news/deep_learning_wizard_nvidia_inception_2018_05_01/" class="md-nav__link">
+        NVIDIA Inception Partner Status, Singapore, May 2017
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+  </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+            
+            
+              
+              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+                    
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+  
+  
+  
+    
+  
+  
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+      
+        <li class="md-nav__item">
+  <a href="#environment-setup" class="md-nav__link">
+    Environment Setup
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#question-and-answer-no-hyperparameter-tuning" class="md-nav__link">
+    Question and Answer | No Hyperparameter Tuning
+  </a>
+  
+    <nav class="md-nav" aria-label="Question and Answer | No Hyperparameter Tuning">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#question-1" class="md-nav__link">
+    Question 1
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#question-2" class="md-nav__link">
+    Question 2
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#question-and-answer-hyperparameter-tuning" class="md-nav__link">
+    Question and Answer | Hyperparameter Tuning
+  </a>
+  
+    <nav class="md-nav" aria-label="Question and Answer | Hyperparameter Tuning">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#temperature-tuning" class="md-nav__link">
+    Temperature Tuning
+  </a>
+  
+    <nav class="md-nav" aria-label="Temperature Tuning">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#low-temperature" class="md-nav__link">
+    Low Temperature
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#high-temperature" class="md-nav__link">
+    High Temperature
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#mathematical-interpretation" class="md-nav__link">
+    Mathematical Interpretation
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#top-k-tuning" class="md-nav__link">
+    Top-K Tuning
+  </a>
+  
+    <nav class="md-nav" aria-label="Top-K Tuning">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#low-k" class="md-nav__link">
+    Low K
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#high-k" class="md-nav__link">
+    High K
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#mathematical-interpretation_1" class="md-nav__link">
+    Mathematical Interpretation
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#summary" class="md-nav__link">
+    Summary
+  </a>
+  
+</li>
+      
+    </ul>
+  
+</nav>
+                  </div>
+                </div>
+              </div>
+            
+          
+          <div class="md-content" data-md-component="content">
+            <article class="md-content__inner md-typeset">
+              
+                
+                  <a href="https://github.com/ritchieng/deep-learning-wizard/edit/master/docs/language_model/llm/llm_intro_hyperparameter_tuning.md" title="Edit this page" class="md-content__button md-icon">
+                    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20.71 7.04c.39-.39.39-1.04 0-1.41l-2.34-2.34c-.37-.39-1.02-.39-1.41 0l-1.84 1.83 3.75 3.75M3 17.25V21h3.75L17.81 9.93l-3.75-3.75L3 17.25z"/></svg>
+                  </a>
+                
+                
+                <h1 id="llm-introduction-hyperparameter-tuning">LLM Introduction &amp; Hyperparameter Tuning<a class="headerlink" href="#llm-introduction-hyperparameter-tuning" title="Permanent link">&para;</a></h1>
+<p>In this tutorial, we will be covering LLMs leveraging on Ollama and LlamaIndex using Gemma:7b (Google) open-source model.</p>
+<h2 id="environment-setup">Environment Setup<a class="headerlink" href="#environment-setup" title="Permanent link">&para;</a></h2>
+<p>Follow our <a href="https://www.deeplearningwizard.com/language_model/containers/hpc_containers_apptainer/">tutorial on Apptainer</a> to get started. Once you have followed the tutorial till the <a href="https://www.deeplearningwizard.com/language_model/containers/hpc_containers_apptainer/#ollama-gemma-workloads">Ollama section</a> where you successfully ran <code>ollama serve</code> and <code>ollama run gemma:7b</code>, you can run the <code>apptainer shell --nv --nvccli apptainer_container_0.1.sif</code> command followed by <code>jupyter lab</code> to access and run this notebook.</p>
+<div class="admonition info">
+<p class="admonition-title">Directory Guide</p>
+<p>When you shell into the Apptainer <code>.sif</code> container, you will need to navigate the directory as you normally would into the Deep Learning Wizard repository that you cloned, requiring you to <code>cd ..</code> to go back a few directories and finally reaching the right folder. </p>
+</div>
+<h2 id="question-and-answer-no-hyperparameter-tuning">Question and Answer | No Hyperparameter Tuning<a class="headerlink" href="#question-and-answer-no-hyperparameter-tuning" title="Permanent link">&para;</a></h2>
+<p>In this section, we will leverage on the <code>Gemma:7b</code> LLM model to ask basic questions to get responses.</p>
+<div class="highlight"><pre><span></span><code><span class="kn">import</span> <span class="nn">ollama</span>
+</code></pre></div>
+<h3 id="question-1">Question 1<a class="headerlink" href="#question-1" title="Permanent link">&para;</a></h3>
+<div class="highlight"><pre><span></span><code><span class="c1"># The &#39;chat&#39; function is called with two parameters: &#39;model&#39; and &#39;messages&#39;.</span>
+<span class="n">response</span> <span class="o">=</span> <span class="n">ollama</span><span class="o">.</span><span class="n">chat</span><span class="p">(</span>
+    <span class="n">model</span><span class="o">=</span><span class="s1">&#39;gemma:7b&#39;</span><span class="p">,</span>  <span class="c1"># The &#39;model&#39; parameter specifies the model to be used. Here, &#39;gemma:7b&#39; is the model.</span>
+    <span class="n">messages</span><span class="o">=</span><span class="p">[</span>  <span class="c1"># The &#39;messages&#39; parameter is a list of message objects.</span>
+        <span class="p">{</span>
+            <span class="s1">&#39;role&#39;</span><span class="p">:</span> <span class="s1">&#39;user&#39;</span><span class="p">,</span>  <span class="c1"># Each message object has a &#39;role&#39; key. It can be &#39;user&#39; or &#39;assistant&#39;.</span>
+            <span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="s1">&#39;What is Singapore?&#39;</span><span class="p">,</span>  <span class="c1"># The &#39;content&#39; key contains the actual content of the message.</span>
+        <span class="p">},</span>
+    <span class="p">]</span>
+<span class="p">)</span>
+
+<span class="c1"># The &#39;chat&#39; function returns a response object. </span>
+<span class="c1"># The content of the assistant&#39;s message is accessed using the keys &#39;message&#39; and &#39;content&#39;.</span>
+<span class="c1"># The &#39;print&#39; function is used to display this content.</span>
+<span class="nb">print</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;message&#39;</span><span class="p">][</span><span class="s1">&#39;content&#39;</span><span class="p">])</span>
+</code></pre></div>
+<div class="highlight"><pre><span></span><code>Singapore is a city-state located on the island of Singapore, a Southeast Asian island. It is a highly developed city known for its modern architecture, efficient transportation system, and vibrant cultural diversity.
+</code></pre></div>
+<h3 id="question-2">Question 2<a class="headerlink" href="#question-2" title="Permanent link">&para;</a></h3>
+<div class="highlight"><pre><span></span><code><span class="n">response</span> <span class="o">=</span> <span class="n">ollama</span><span class="o">.</span><span class="n">chat</span><span class="p">(</span>
+    <span class="n">model</span><span class="o">=</span><span class="s1">&#39;gemma:7b&#39;</span><span class="p">,</span>
+    <span class="n">messages</span><span class="o">=</span><span class="p">[</span>
+        <span class="p">{</span>
+            <span class="s1">&#39;role&#39;</span><span class="p">:</span> <span class="s1">&#39;user&#39;</span><span class="p">,</span>
+            <span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="s1">&#39;What is a Large Language Model?&#39;</span><span class="p">,</span>
+        <span class="p">},</span>
+    <span class="p">]</span>
+<span class="p">)</span>
+
+<span class="nb">print</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;message&#39;</span><span class="p">][</span><span class="s1">&#39;content&#39;</span><span class="p">])</span>
+</code></pre></div>
+<div class="highlight"><pre><span></span><code>Sure, a Large Language Model (LLM) is a type of language model that has been trained on a massive amount of text data and has the ability to engage in a wide range of natural language processing tasks. LLMs are typically designed to have a large number of parameters, which allows them to learn complex relationships between words and sentences. LLMs are often used for tasks such as text summarization, translation, and code generation.
+</code></pre></div>
+<p>In our second question, we change the question to "What is a Large Language Model?" and you can observe how the answer is slightly longer than the first question "What is Singapore". In the next section, you will discover that this relates to a few hyperparemeters in LLMs that can be tweaked.</p>
+<h2 id="question-and-answer-hyperparameter-tuning">Question and Answer | Hyperparameter Tuning<a class="headerlink" href="#question-and-answer-hyperparameter-tuning" title="Permanent link">&para;</a></h2>
+<h3 id="temperature-tuning">Temperature Tuning<a class="headerlink" href="#temperature-tuning" title="Permanent link">&para;</a></h3>
+<p>The <code>temperature</code> parameter in LLMs plays a pivotal role in determining the predictability of the output. </p>
+<ul>
+<li>
+<p><strong>Lower temperature values (e.g., 0.2)</strong> lead to more predictable and consistent responses, but may risk being overly constrained or repetitive.</p>
+</li>
+<li>
+<p><strong>Higher temperature values (e.g., 1.0)</strong> introduce more randomness and diversity, but can result in less consistency and occasional incoherence.
+The choice of temperature value is a trade-off between consistency and variety, and should be tailored to the specific requirements of your task.</p>
+</li>
+</ul>
+<h4 id="low-temperature">Low Temperature<a class="headerlink" href="#low-temperature" title="Permanent link">&para;</a></h4>
+<div class="highlight"><pre><span></span><code><span class="c1"># Create new model</span>
+<span class="c1"># Set the temperature</span>
+<span class="c1"># Higher values (e.g., 1.0) make the output more random, while lower values (e.g., 0.1) make it more deterministic</span>
+<span class="n">modelfile</span><span class="o">=</span><span class="s1">&#39;&#39;&#39;</span>
+<span class="s1">FROM gemma:7b</span>
+<span class="s1">PARAMETER temperature 0.1</span>
+<span class="s1">&#39;&#39;&#39;</span>
+<span class="n">ollama</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="s1">&#39;gemma_low_temp&#39;</span><span class="p">,</span> <span class="n">modelfile</span><span class="o">=</span><span class="n">modelfile</span><span class="p">)</span>
+
+<span class="c1"># Now you can use the new model with adjusted temperature</span>
+<span class="n">response</span> <span class="o">=</span> <span class="n">ollama</span><span class="o">.</span><span class="n">chat</span><span class="p">(</span>
+    <span class="n">model</span><span class="o">=</span><span class="s1">&#39;gemma_low_temp&#39;</span><span class="p">,</span>
+    <span class="n">messages</span><span class="o">=</span><span class="p">[</span>
+        <span class="p">{</span>
+            <span class="s1">&#39;role&#39;</span><span class="p">:</span> <span class="s1">&#39;user&#39;</span><span class="p">,</span>
+            <span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="s1">&#39;Write a happy birthday message, I would like to send to my friend.&#39;</span><span class="p">,</span>
+        <span class="p">},</span>
+    <span class="p">]</span>
+<span class="p">)</span>
+
+<span class="nb">print</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;message&#39;</span><span class="p">][</span><span class="s1">&#39;content&#39;</span><span class="p">])</span>
+</code></pre></div>
+<div class="highlight"><pre><span></span><code>Happy Birthday, [Friend&#39;s Name]! I hope your day is filled with joy, laughter, and happiness. May all your wishes come true! 🎉🎂
+</code></pre></div>
+<div class="highlight"><pre><span></span><code><span class="c1"># Run multiple times</span>
+<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">3</span><span class="p">):</span>
+    <span class="n">response</span> <span class="o">=</span> <span class="n">ollama</span><span class="o">.</span><span class="n">chat</span><span class="p">(</span>
+        <span class="n">model</span><span class="o">=</span><span class="s1">&#39;gemma_low_temp&#39;</span><span class="p">,</span>
+        <span class="n">messages</span><span class="o">=</span><span class="p">[</span>
+            <span class="p">{</span>
+                <span class="s1">&#39;role&#39;</span><span class="p">:</span> <span class="s1">&#39;user&#39;</span><span class="p">,</span>
+                <span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="s1">&#39;Write a happy birthday message, I would like to send to my friend.&#39;</span><span class="p">,</span>
+            <span class="p">},</span>
+        <span class="p">],</span>
+    <span class="p">)</span>
+
+    <span class="c1"># Print </span>
+    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="o">*</span><span class="mi">10</span><span class="p">)</span>
+    <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Response </span><span class="si">{</span><span class="n">i</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span> 
+    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="o">*</span><span class="mi">10</span><span class="p">)</span>
+    <span class="nb">print</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;message&#39;</span><span class="p">][</span><span class="s1">&#39;content&#39;</span><span class="p">])</span>
+</code></pre></div>
+<div class="highlight"><pre><span></span><code>----------
+Response 0
+----------
+Happy Birthday, [Friend&#39;s Name]! I hope your day is filled with joy, laughter, and happiness. May all your wishes come true! 🎉🎂
+----------
+Response 1
+----------
+Happy Birthday, [Friend&#39;s Name]! I hope your day is filled with joy, laughter, and happiness. May all your wishes come true! 🎉🎂
+----------
+Response 2
+----------
+Happy Birthday, [Friend&#39;s Name]! I hope your day is filled with joy, laughter, and happiness. May all your wishes come true! 🎉🎂
+</code></pre></div>
+<p><strong>We can see above it is the exact same answer calling the LLM 3 times</strong>.</p>
+<h4 id="high-temperature">High Temperature<a class="headerlink" href="#high-temperature" title="Permanent link">&para;</a></h4>
+<div class="highlight"><pre><span></span><code><span class="c1"># Create new model</span>
+<span class="c1"># Set the temperature</span>
+<span class="c1"># Higher values (e.g., 1.0) make the output more random, while lower values (e.g., 0.1) make it more deterministic</span>
+<span class="n">modelfile</span><span class="o">=</span><span class="s1">&#39;&#39;&#39;</span>
+<span class="s1">FROM gemma:7b</span>
+<span class="s1">PARAMETER temperature 1.0</span>
+<span class="s1">&#39;&#39;&#39;</span>
+<span class="n">ollama</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="s1">&#39;gemma_high_temp&#39;</span><span class="p">,</span> <span class="n">modelfile</span><span class="o">=</span><span class="n">modelfile</span><span class="p">)</span>
+
+<span class="c1"># Now you can use the new model with adjusted temperature</span>
+<span class="n">response</span> <span class="o">=</span> <span class="n">ollama</span><span class="o">.</span><span class="n">chat</span><span class="p">(</span>
+    <span class="n">model</span><span class="o">=</span><span class="s1">&#39;gemma_high_temp&#39;</span><span class="p">,</span>
+    <span class="n">messages</span><span class="o">=</span><span class="p">[</span>
+        <span class="p">{</span>
+            <span class="s1">&#39;role&#39;</span><span class="p">:</span> <span class="s1">&#39;user&#39;</span><span class="p">,</span>
+            <span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="s1">&#39;Write a happy birthday message, I would like to send to my friend.&#39;</span><span class="p">,</span>
+        <span class="p">},</span>
+    <span class="p">]</span>
+<span class="p">)</span>
+
+<span class="nb">print</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;message&#39;</span><span class="p">][</span><span class="s1">&#39;content&#39;</span><span class="p">])</span>
+</code></pre></div>
+<div class="highlight"><pre><span></span><code>Happy Birthday, [Friend&#39;s Name]! I hope your day is filled with joy, laughter, and happiness! 🎉🎂 🎉
+</code></pre></div>
+<div class="highlight"><pre><span></span><code><span class="c1"># Run multiple times</span>
+<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">3</span><span class="p">):</span>
+    <span class="n">response</span> <span class="o">=</span> <span class="n">ollama</span><span class="o">.</span><span class="n">chat</span><span class="p">(</span>
+        <span class="n">model</span><span class="o">=</span><span class="s1">&#39;gemma_high_temp&#39;</span><span class="p">,</span>
+        <span class="n">messages</span><span class="o">=</span><span class="p">[</span>
+            <span class="p">{</span>
+                <span class="s1">&#39;role&#39;</span><span class="p">:</span> <span class="s1">&#39;user&#39;</span><span class="p">,</span>
+                <span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="s1">&#39;Write a happy birthday message, I would like to send to my friend.&#39;</span><span class="p">,</span>
+            <span class="p">},</span>
+        <span class="p">],</span>
+    <span class="p">)</span>
+
+    <span class="c1"># Print </span>
+    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="o">*</span><span class="mi">10</span><span class="p">)</span>
+    <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Response </span><span class="si">{</span><span class="n">i</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span> 
+    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="o">*</span><span class="mi">10</span><span class="p">)</span>
+    <span class="nb">print</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;message&#39;</span><span class="p">][</span><span class="s1">&#39;content&#39;</span><span class="p">])</span>
+</code></pre></div>
+<div class="highlight"><pre><span></span><code>----------
+Response 0
+----------
+Happy Birthday, [Friend&#39;s Name]! I hope your day is filled with joy, laughter, and happiness. Let&#39;s celebrate your special day together!
+----------
+Response 1
+----------
+Happy Birthday, [Friend&#39;s Name]! I hope your day is filled with joy, laughter, and happiness. Wishing you a very special day filled with memorable moments and sweet treats!
+----------
+Response 2
+----------
+Sure, here is a happy birthday message you can send to your friend:
+
+**Happy Birthday, [Friend&#39;s Name]!**
+
+May your day be filled with joy, laughter, and happiness. I hope your special day is filled with everything you wish for. I&#39;m sending you positive vibes and can&#39;t wait to see you soon.
+</code></pre></div>
+<p><strong>Here, you can see very varied answer in each of the 3 calls to the LLM compared to lower temperature.</strong></p>
+<h4 id="mathematical-interpretation">Mathematical Interpretation<a class="headerlink" href="#mathematical-interpretation" title="Permanent link">&para;</a></h4>
+<p>In LLMs, the <code>temperature</code> parameter is used to control the randomness of predictions by scaling the logits before applying soft(arg)max.</p>
+<ul>
+<li>
+<p>The model computes a score (also known as logits) for each possible next token based on the current context. These scores are then transformed into probabilities using the soft(arg)max function.</p>
+</li>
+<li>
+<p>The soft(arg)max function is defined as follows:</p>
+<div class="arithmatex">\[\text{softargmax}(x_i) = \frac{e^{x_i}}{\sum_i^j e^{x_i}}\]</div>
+</li>
+<li>
+<p>Before applying soft(arg)max, the logits are divided by the <code>temperature</code> value. This process is called temperature scaling and the equation becomes:</p>
+<div class="arithmatex">\[\text{softargmax}(x_i) = \frac{e^{x_i/T}}{\sum_i^j e^{x_i/T}}\]</div>
+</li>
+<li>
+<p>When <code>T</code> &gt; 1, it makes the distribution more uniform (increases randomness). When <code>T</code> &lt; 1, it makes the distribution more peaky (reduces randomness).</p>
+</li>
+</ul>
+<p>So, in simple terms, a higher temperature value makes the model's output more random, and a lower temperature makes it more deterministic.</p>
+<div class="highlight"><pre><span></span><code><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
+
+<span class="k">def</span> <span class="nf">softargmax</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">T</span><span class="o">=</span><span class="mf">1.0</span><span class="p">):</span>
+    <span class="n">e_x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="n">x</span> <span class="o">/</span> <span class="n">T</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">e_x</span> <span class="o">/</span> <span class="n">e_x</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span>
+
+<span class="c1"># Define logits</span>
+<span class="n">logits</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mf">0.2</span><span class="p">,</span> <span class="mf">0.3</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.4</span><span class="p">])</span>
+
+<span class="c1"># Compute soft(arg)max for different temperatures</span>
+<span class="k">for</span> <span class="n">T</span> <span class="ow">in</span> <span class="p">[</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">]:</span>
+    <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Temperature: </span><span class="si">{</span><span class="n">T</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+    <span class="nb">print</span><span class="p">(</span><span class="n">softargmax</span><span class="p">(</span><span class="n">logits</span><span class="p">,</span> <span class="n">T</span><span class="o">=</span><span class="n">T</span><span class="p">))</span>
+    <span class="nb">print</span><span class="p">()</span>
+</code></pre></div>
+<div class="highlight"><pre><span></span><code>Temperature: 0.1
+[0.08714432 0.23688282 0.0320586  0.64391426]
+
+Temperature: 1.0
+[0.23632778 0.26118259 0.21383822 0.28865141]
+</code></pre></div>
+<p>In the Python code above leveraging on <code>numpy</code> library, you can see that</p>
+<ul>
+<li><code>softargmax</code> is a function that computes the soft(arg)max of an array of logits <code>x</code> for a given temperature <code>T</code>.</li>
+<li>We define an array of logits and compute the soft(arg)max for different temperatures.</li>
+<li>
+<p>When you run this code, you'll see that as the temperature increases, the soft(arg)max output becomes more uniform (i.e., the probabilities are more evenly distributed), and as the temperature decreases, the soft(arg)max output becomes more peaky (i.e., one probability dominates the others). This illustrates how temperature can control the randomness of the model's output.</p>
+</li>
+<li>
+<p>To close this off, taking the max of the soft(arg)max output, you will observe how it gets more random in the max value as the soft(arg)max output becomes more uniform. This links to the concept of how the next word gets more random because of the max of the uniformity of the soft(arg)max output.</p>
+</li>
+</ul>
+<h3 id="top-k-tuning">Top-K Tuning<a class="headerlink" href="#top-k-tuning" title="Permanent link">&para;</a></h3>
+<p>In LLMs, the <code>top_k</code> hyperparameter is a key factor that influences the unpredictability of the generated output.</p>
+<ul>
+<li>
+<p><strong>For smaller <code>top_k</code> values</strong>: The model behaves in a more predictable manner. It only takes into account a limited set of the most probable next tokens at each step of the generation process. This can result in responses that are more concise and consistent, but there’s a possibility that the output may be too restricted or repetitive.</p>
+</li>
+<li>
+<p><strong>For larger <code>top_k</code> values</strong>: The model takes into consideration a broader set of potential next tokens. This infuses more variety and randomness into the generated output. However, the responses can become less consistent and may occasionally be less coherent or pertinent.
+Therefore, the selection of the top_k value can be viewed as a balance between consistency and variety in the model’s responses. It’s crucial to adjust this parameter based on the specific needs of your task. </p>
+</li>
+</ul>
+<h4 id="low-k">Low K<a class="headerlink" href="#low-k" title="Permanent link">&para;</a></h4>
+<div class="highlight"><pre><span></span><code><span class="c1"># Create new model</span>
+
+<span class="c1"># Set the temperature</span>
+<span class="c1"># Higher values (e.g., 1.0) make the output more random, while lower values (e.g., 0.1) make it more deterministic</span>
+
+<span class="c1"># Set the top_k</span>
+<span class="c1"># This parameter controls the number of tokens considered for each step of the generation process</span>
+<span class="n">modelfile</span><span class="o">=</span><span class="s1">&#39;&#39;&#39;</span>
+<span class="s1">FROM gemma:7b</span>
+<span class="s1">PARAMETER temperature 0.5</span>
+<span class="s1">PARAMETER top_k 3</span>
+<span class="s1">&#39;&#39;&#39;</span>
+<span class="n">ollama</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="s1">&#39;gemma_topk_3&#39;</span><span class="p">,</span> <span class="n">modelfile</span><span class="o">=</span><span class="n">modelfile</span><span class="p">)</span>
+
+<span class="c1"># Now you can use the new model with adjusted temperature</span>
+<span class="n">response</span> <span class="o">=</span> <span class="n">ollama</span><span class="o">.</span><span class="n">chat</span><span class="p">(</span>
+    <span class="n">model</span><span class="o">=</span><span class="s1">&#39;gemma_topk_3&#39;</span><span class="p">,</span>
+    <span class="n">messages</span><span class="o">=</span><span class="p">[</span>
+        <span class="p">{</span>
+            <span class="s1">&#39;role&#39;</span><span class="p">:</span> <span class="s1">&#39;user&#39;</span><span class="p">,</span>
+            <span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="s1">&#39;Write a happy birthday message, I would like to send to my friend.&#39;</span><span class="p">,</span>
+        <span class="p">},</span>
+    <span class="p">]</span>
+<span class="p">)</span>
+
+<span class="nb">print</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;message&#39;</span><span class="p">][</span><span class="s1">&#39;content&#39;</span><span class="p">])</span>
+</code></pre></div>
+<div class="highlight"><pre><span></span><code>Happy Birthday, [Friend&#39;s Name]! I hope your day is filled with joy, laughter, and happiness. May all your wishes come true!
+</code></pre></div>
+<div class="highlight"><pre><span></span><code><span class="c1"># Run multiple times</span>
+<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">3</span><span class="p">):</span>
+    <span class="n">response</span> <span class="o">=</span> <span class="n">ollama</span><span class="o">.</span><span class="n">chat</span><span class="p">(</span>
+        <span class="n">model</span><span class="o">=</span><span class="s1">&#39;gemma_topk_3&#39;</span><span class="p">,</span>
+        <span class="n">messages</span><span class="o">=</span><span class="p">[</span>
+            <span class="p">{</span>
+                <span class="s1">&#39;role&#39;</span><span class="p">:</span> <span class="s1">&#39;user&#39;</span><span class="p">,</span>
+                <span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="s1">&#39;Write a happy birthday message, I would like to send to my friend.&#39;</span><span class="p">,</span>
+            <span class="p">},</span>
+        <span class="p">],</span>
+    <span class="p">)</span>
+
+    <span class="c1"># Print </span>
+    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="o">*</span><span class="mi">10</span><span class="p">)</span>
+    <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Response </span><span class="si">{</span><span class="n">i</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span> 
+    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="o">*</span><span class="mi">10</span><span class="p">)</span>
+    <span class="nb">print</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;message&#39;</span><span class="p">][</span><span class="s1">&#39;content&#39;</span><span class="p">])</span>
+</code></pre></div>
+<div class="highlight"><pre><span></span><code>----------
+Response 0
+----------
+Sure, here&#39;s a happy birthday message for your friend:
+
+**Happy Birthday, [Friend&#39;s Name]!**
+
+I hope your day is filled with joy, laughter, and happiness. May all your wishes come true. Have a blast! 🎉🎂🎉
+----------
+Response 1
+----------
+Happy Birthday, [Friend&#39;s Name]! I hope your day is filled with joy, laughter, and happiness. May all your wishes come true. 🎉🎂
+----------
+Response 2
+----------
+Happy Birthday, [Friend&#39;s Name]! I hope your day is filled with joy, laughter, and happiness. May all your wishes come true. 🎉🎂
+</code></pre></div>
+<h4 id="high-k">High K<a class="headerlink" href="#high-k" title="Permanent link">&para;</a></h4>
+<div class="highlight"><pre><span></span><code><span class="c1"># Create new model</span>
+
+<span class="c1"># Set the temperature</span>
+<span class="c1"># Higher values (e.g., 1.0) make the output more random, while lower values (e.g., 0.1) make it more deterministic</span>
+
+<span class="c1"># Set the top_k</span>
+<span class="c1"># This parameter controls the number of tokens considered for each step of the generation process</span>
+<span class="n">modelfile</span><span class="o">=</span><span class="s1">&#39;&#39;&#39;</span>
+<span class="s1">FROM gemma:7b</span>
+<span class="s1">PARAMETER temperature 0.5</span>
+<span class="s1">PARAMETER top_k 200</span>
+<span class="s1">&#39;&#39;&#39;</span>
+<span class="n">ollama</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="s1">&#39;gemma_topk_200&#39;</span><span class="p">,</span> <span class="n">modelfile</span><span class="o">=</span><span class="n">modelfile</span><span class="p">)</span>
+
+<span class="c1"># Now you can use the new model with adjusted temperature</span>
+<span class="n">response</span> <span class="o">=</span> <span class="n">ollama</span><span class="o">.</span><span class="n">chat</span><span class="p">(</span>
+    <span class="n">model</span><span class="o">=</span><span class="s1">&#39;gemma_topk_200&#39;</span><span class="p">,</span>
+    <span class="n">messages</span><span class="o">=</span><span class="p">[</span>
+        <span class="p">{</span>
+            <span class="s1">&#39;role&#39;</span><span class="p">:</span> <span class="s1">&#39;user&#39;</span><span class="p">,</span>
+            <span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="s1">&#39;Write a happy birthday message, I would like to send to my friend.&#39;</span><span class="p">,</span>
+        <span class="p">},</span>
+    <span class="p">]</span>
+<span class="p">)</span>
+
+<span class="nb">print</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;message&#39;</span><span class="p">][</span><span class="s1">&#39;content&#39;</span><span class="p">])</span>
+</code></pre></div>
+<div class="highlight"><pre><span></span><code>Happy Birthday, [Friend&#39;s name]! I hope your day is filled with joy, laughter, and happiness! 🎉🎂
+</code></pre></div>
+<div class="highlight"><pre><span></span><code><span class="c1"># Run multiple times</span>
+<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">3</span><span class="p">):</span>
+    <span class="n">response</span> <span class="o">=</span> <span class="n">ollama</span><span class="o">.</span><span class="n">chat</span><span class="p">(</span>
+        <span class="n">model</span><span class="o">=</span><span class="s1">&#39;gemma_topk_200&#39;</span><span class="p">,</span>
+        <span class="n">messages</span><span class="o">=</span><span class="p">[</span>
+            <span class="p">{</span>
+                <span class="s1">&#39;role&#39;</span><span class="p">:</span> <span class="s1">&#39;user&#39;</span><span class="p">,</span>
+                <span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="s1">&#39;Write a happy birthday message, I would like to send to my friend.&#39;</span><span class="p">,</span>
+            <span class="p">},</span>
+        <span class="p">],</span>
+    <span class="p">)</span>
+
+    <span class="c1"># Print </span>
+    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="o">*</span><span class="mi">10</span><span class="p">)</span>
+    <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Response </span><span class="si">{</span><span class="n">i</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span> 
+    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="o">*</span><span class="mi">10</span><span class="p">)</span>
+    <span class="nb">print</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;message&#39;</span><span class="p">][</span><span class="s1">&#39;content&#39;</span><span class="p">])</span>
+</code></pre></div>
+<div class="highlight"><pre><span></span><code>----------
+Response 0
+----------
+Sure, here is a happy birthday message for your friend:
+
+**Happy Birthday, [Friend&#39;s Name]!**
+
+I hope your day is filled with joy, laughter, and happiness. May all your wishes come true today.
+
+Have a wonderful birthday, and I look forward to seeing you soon.
+
+**Best regards,**
+
+[Your Name]
+----------
+Response 1
+----------
+Sure, here&#39;s a happy birthday message for your friend:
+
+**Happy Birthday, [Friend&#39;s Name]!**
+
+May your day be filled with joy, laughter, and happiness. I hope your special day is filled with all your favorite things and that your wishes come true.
+
+**Have a wonderful birthday, my dear friend!**
+----------
+Response 2
+----------
+Sure, here&#39;s a happy birthday message for your friend:
+
+**Happy Birthday, [Friend&#39;s Name]!**
+
+May your day be filled with joy, laughter, and happiness. I hope you have a wonderful time celebrating your special day!
+</code></pre></div>
+<p>You can observe that the reply is more diverse with a high <code>top_k</code> hyperparameter.</p>
+<h4 id="mathematical-interpretation_1">Mathematical Interpretation<a class="headerlink" href="#mathematical-interpretation_1" title="Permanent link">&para;</a></h4>
+<p>In LLMs, the <code>top_k</code> parameter is used to limit the number of next tokens considered for generation.</p>
+<ul>
+<li>
+<p>After computing the soft(arg)max probabilities for all possible next tokens, the model sorts these probabilities in descending order.</p>
+</li>
+<li>
+<p>The model then only considers the <code>top_k</code> tokens with the highest probabilities for the next step of the generation process.</p>
+</li>
+<li>
+<p>This process is called <code>top_k</code> sampling.</p>
+</li>
+</ul>
+<p>Here's a simple Python code snippet that illustrates how <code>top_k</code> works.</p>
+<div class="highlight"><pre><span></span><code><span class="k">def</span> <span class="nf">top_k</span><span class="p">(</span><span class="n">logits</span><span class="p">,</span> <span class="n">k</span><span class="p">):</span>
+    <span class="c1"># Sort the logits</span>
+    <span class="n">sorted_indices</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">argsort</span><span class="p">(</span><span class="n">logits</span><span class="p">)</span>
+
+    <span class="c1"># Consider only the top k</span>
+    <span class="n">top_k_indices</span> <span class="o">=</span> <span class="n">sorted_indices</span><span class="p">[</span><span class="o">-</span><span class="n">k</span><span class="p">:]</span>
+
+    <span class="c1"># Create a new array with only the top k probabilities</span>
+    <span class="n">top_k_logits</span> <span class="o">=</span> <span class="n">logits</span><span class="p">[</span><span class="n">top_k_indices</span><span class="p">]</span>
+
+    <span class="k">return</span> <span class="n">top_k_logits</span>
+
+<span class="c1"># Define logits</span>
+<span class="n">logits</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mf">0.2</span><span class="p">,</span> <span class="mf">0.3</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.4</span><span class="p">])</span>
+
+<span class="c1"># Compute top_k for different values of k</span>
+<span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]:</span>
+    <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Top </span><span class="si">{</span><span class="n">k</span><span class="si">}</span><span class="s2"> logits:&quot;</span><span class="p">)</span>
+    <span class="nb">print</span><span class="p">(</span><span class="n">top_k</span><span class="p">(</span><span class="n">logits</span><span class="p">,</span> <span class="n">k</span><span class="o">=</span><span class="n">k</span><span class="p">))</span>
+    <span class="nb">print</span><span class="p">()</span>
+</code></pre></div>
+<div class="highlight"><pre><span></span><code>Top 2 logits:
+[0.3 0.4]
+
+Top 3 logits:
+[0.2 0.3 0.4]
+
+Top 4 logits:
+[0.1 0.2 0.3 0.4]
+</code></pre></div>
+<p>In the code above</p>
+<ul>
+<li>
+<p><code>top_k</code> is a function that computes the top <code>k</code> logits from an array of logits.</p>
+</li>
+<li>
+<p>We define an array of logits and compute the top <code>k</code> logits for different values of <code>k</code>.</p>
+</li>
+<li>
+<p>When you run this code, you'll see that as <code>k</code> increases, more logits are considered. This illustrates how <code>top_k</code> can control the number of tokens considered by the model.</p>
+</li>
+</ul>
+<h2 id="summary">Summary<a class="headerlink" href="#summary" title="Permanent link">&para;</a></h2>
+<p>We covered the functionality of a basic LLM without any hyperparameter tuning. We then covered Temperature and Top-K hyperparameter tuning. It is important to note that there are many hyperparameters that can be tuned, and we will update this tutorial to gradually include as many as we can.</p>
+                
+              
+              
+                
+
+
+  <h2 id="__comments">Comments</h2>
+  <div id="disqus_thread"></div>
+  <script>var disqus_config=function(){this.page.url="https://www.deeplearningwizard.com/language_model/llm/llm_intro_hyperparameter_tuning/",this.page.identifier="language_model/llm/llm_intro_hyperparameter_tuning/"};window.addEventListener("load",function(){var e=document,i=e.createElement("script");i.src="//deep-learning-wizard.disqus.com/embed.js",i.setAttribute("data-timestamp",+new Date),(e.head||e.body).appendChild(i)})</script>
+
+              
+            </article>
+          </div>
+        </div>
+        
+          <a href="#" class="md-top md-icon" data-md-component="top" data-md-state="hidden">
+            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12z"/></svg>
+            Back to top
+          </a>
+        
+      </main>
+      
+        
+<footer class="md-footer">
+  
+    <nav class="md-footer__inner md-grid" aria-label="Footer">
+      
+        
+        <a href="../../containers/hpc_containers_apptainer/" class="md-footer__link md-footer__link--prev" aria-label="Previous: HPC Containers with Apptainer" rel="prev">
+          <div class="md-footer__button md-icon">
+            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12z"/></svg>
+          </div>
+          <div class="md-footer__title">
+            <div class="md-ellipsis">
+              <span class="md-footer__direction">
+                Previous
+              </span>
+              HPC Containers with Apptainer
+            </div>
+          </div>
+        </a>
+      
+      
+        
+        <a href="../../../machine_learning/intro/" class="md-footer__link md-footer__link--next" aria-label="Next: Introduction" rel="next">
+          <div class="md-footer__title">
+            <div class="md-ellipsis">
+              <span class="md-footer__direction">
+                Next
+              </span>
+              Introduction
+            </div>
+          </div>
+          <div class="md-footer__button md-icon">
+            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11H4z"/></svg>
+          </div>
+        </a>
+      
+    </nav>
+  
+  <div class="md-footer-meta md-typeset">
+    <div class="md-footer-meta__inner md-grid">
+      <div class="md-footer-copyright">
+        
+          <div class="md-footer-copyright__highlight">
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
+          </div>
+        
+        
+      </div>
+      
+  <div class="md-footer-social">
+    
+      
+      
+        
+        
+      
+      <a href="https://www.youtube.com/channel/UCJz2MIjiCosOQCwhnsYxeEw" target="_blank" rel="noopener" title="www.youtube.com" class="md-footer-social__link">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512"><path d="M549.655 124.083c-6.281-23.65-24.787-42.276-48.284-48.597C458.781 64 288 64 288 64S117.22 64 74.629 75.486c-23.497 6.322-42.003 24.947-48.284 48.597-11.412 42.867-11.412 132.305-11.412 132.305s0 89.438 11.412 132.305c6.281 23.65 24.787 41.5 48.284 47.821C117.22 448 288 448 288 448s170.78 0 213.371-11.486c23.497-6.321 42.003-24.171 48.284-47.821 11.412-42.867 11.412-132.305 11.412-132.305s0-89.438-11.412-132.305zm-317.51 213.508V175.185l142.739 81.205-142.739 81.201z"/></svg>
+      </a>
+    
+      
+      
+        
+        
+      
+      <a href="https://twitter.com/deeplearningwiz" target="_blank" rel="noopener" title="twitter.com" class="md-footer-social__link">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M459.37 151.716c.325 4.548.325 9.097.325 13.645 0 138.72-105.583 298.558-298.558 298.558-59.452 0-114.68-17.219-161.137-47.106 8.447.974 16.568 1.299 25.34 1.299 49.055 0 94.213-16.568 130.274-44.832-46.132-.975-84.792-31.188-98.112-72.772 6.498.974 12.995 1.624 19.818 1.624 9.421 0 18.843-1.3 27.614-3.573-48.081-9.747-84.143-51.98-84.143-102.985v-1.299c13.969 7.797 30.214 12.67 47.431 13.319-28.264-18.843-46.781-51.005-46.781-87.391 0-19.492 5.197-37.36 14.294-52.954 51.655 63.675 129.3 105.258 216.365 109.807-1.624-7.797-2.599-15.918-2.599-24.04 0-57.828 46.782-104.934 104.934-104.934 30.213 0 57.502 12.67 76.67 33.137 23.715-4.548 46.456-13.32 66.599-25.34-7.798 24.366-24.366 44.833-46.132 57.827 21.117-2.273 41.584-8.122 60.426-16.243-14.292 20.791-32.161 39.308-52.628 54.253z"/></svg>
+      </a>
+    
+      
+      
+        
+        
+      
+      <a href="https://www.facebook.com/DeepLearningWizard/" target="_blank" rel="noopener" title="www.facebook.com" class="md-footer-social__link">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M504 256C504 119 393 8 256 8S8 119 8 256c0 123.78 90.69 226.38 209.25 245V327.69h-63V256h63v-54.64c0-62.15 37-96.48 93.67-96.48 27.14 0 55.52 4.84 55.52 4.84v61h-31.28c-30.8 0-40.41 19.12-40.41 38.73V256h68.78l-11 71.69h-57.78V501C413.31 482.38 504 379.78 504 256z"/></svg>
+      </a>
+    
+      
+      
+        
+        
+      
+      <a href="https://www.linkedin.com/company/deeplearningwizard/" target="_blank" rel="noopener" title="www.linkedin.com" class="md-footer-social__link">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M416 32H31.9C14.3 32 0 46.5 0 64.3v383.4C0 465.5 14.3 480 31.9 480H416c17.6 0 32-14.5 32-32.3V64.3c0-17.8-14.4-32.3-32-32.3zM135.4 416H69V202.2h66.5V416zm-33.2-243c-21.3 0-38.5-17.3-38.5-38.5S80.9 96 102.2 96c21.2 0 38.5 17.3 38.5 38.5 0 21.3-17.2 38.5-38.5 38.5zm282.1 243h-66.4V312c0-24.8-.5-56.7-34.5-56.7-34.6 0-39.9 27-39.9 54.9V416h-66.4V202.2h63.7v29.2h.9c8.9-16.8 30.6-34.5 62.9-34.5 67.2 0 79.7 44.3 79.7 101.9V416z"/></svg>
+      </a>
+    
+      
+      
+        
+        
+      
+      <a href="https://github.com/ritchieng" target="_blank" rel="noopener" title="github.com" class="md-footer-social__link">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
+      </a>
+    
+  </div>
+
+    </div>
+  </div>
+</footer>
+      
+    </div>
+    <div class="md-dialog" data-md-component="dialog">
+      <div class="md-dialog__inner md-typeset"></div>
+    </div>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
+    
+    
+      <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
+      
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
+      
+    
+  </body>
+</html>
\ No newline at end of file
diff --git a/machine_learning/gpu/gpu_fractional_differencing/index.html b/machine_learning/gpu/gpu_fractional_differencing/index.html
index 78fa57de6..765c03c9c 100644
--- a/machine_learning/gpu/gpu_fractional_differencing/index.html
+++ b/machine_learning/gpu/gpu_fractional_differencing/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/machine_learning/gpu/gpu_fractional_differencing/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -256,6 +256,22 @@
         
   
   
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
     
   
 
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -908,6 +924,130 @@
 
   
   
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
+      
+      <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
     
   
   
@@ -915,14 +1055,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" checked>
       
-      <label class="md-nav__link" for="__nav_4">
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -952,14 +1092,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" checked>
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1284,14 +1424,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1422,14 +1562,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1445,14 +1585,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1505,14 +1645,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -2177,7 +2317,7 @@ <h3 id="fractional-differencing-weight-function-and-plot">Fractional Differencin
 <li>Weight converges to zero: <span class="arithmatex">\(w_k \rightarrow 0\)</span></li>
 </ul>
 <div class="highlight"><pre><span></span><code><span class="k">def</span> <span class="nf">get_weights</span><span class="p">(</span><span class="n">d</span><span class="p">,</span> <span class="n">num_k</span><span class="p">):</span>
-    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Calculate weights ($w$) for each lag ($k$) through</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Calculate weights ($w$) for each lag ($k$) through</span>
 <span class="sd">    $w_k = -w_{k-1} \frac{d - k + 1}{k}$.</span>
 
 <span class="sd">    Args:</span>
@@ -2194,7 +2334,7 @@ <h3 id="fractional-differencing-weight-function-and-plot">Fractional Differencin
     <span class="k">return</span> <span class="n">w_k</span>
 
 <span class="k">def</span> <span class="nf">plot_weights</span><span class="p">(</span><span class="n">range_d</span><span class="p">,</span> <span class="n">num_k</span><span class="p">,</span> <span class="n">num_d_interval</span><span class="p">):</span>
-    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Plot weights ($w$) for each lag ($k$) for varying differencing value ($d$).</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Plot weights ($w$) for each lag ($k$) for varying differencing value ($d$).</span>
 
 <span class="sd">    Args:</span>
 <span class="sd">        range_d (list): range of differencing values to plot.</span>
@@ -2416,7 +2556,7 @@ <h3 id="floored-weights-function">Floored Weights Function<a class="headerlink"
 </li>
 </ul>
 <div class="highlight"><pre><span></span><code><span class="k">def</span> <span class="nf">get_weights_floored</span><span class="p">(</span><span class="n">d</span><span class="p">,</span> <span class="n">num_k</span><span class="p">,</span> <span class="n">floor</span><span class="o">=</span><span class="mf">1e-3</span><span class="p">):</span>
-    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Calculate weights ($w$) for each lag ($k$) through</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Calculate weights ($w$) for each lag ($k$) through</span>
 <span class="sd">    $w_k = -w_{k-1} \frac{d - k + 1}{k}$ provided weight above a minimum value</span>
 <span class="sd">    (floor) for the weights to prevent computation of weights for the entire</span>
 <span class="sd">    time series.</span>
@@ -2464,7 +2604,7 @@ <h3 id="fixed-window-fractional-differencing-function-cpu">Fixed Window Fraction
 </li>
 </ul>
 <div class="highlight"><pre><span></span><code><span class="k">def</span> <span class="nf">frac_diff</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">d</span><span class="p">,</span> <span class="n">floor</span><span class="o">=</span><span class="mf">1e-3</span><span class="p">):</span>
-    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Fractionally difference time series via CPU.</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Fractionally difference time series via CPU.</span>
 
 <span class="sd">    Args:</span>
 <span class="sd">        df (pd.DataFrame): dataframe of raw time series values.</span>
@@ -2541,7 +2681,7 @@ <h3 id="fixed-window-fractional-differencing-function-gpu">Fixed Window Fraction
         <span class="n">out</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">rolling_dot_product</span> 
 
 <span class="k">def</span> <span class="nf">frac_diff_gpu</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">d</span><span class="p">,</span> <span class="n">floor</span><span class="o">=</span><span class="mf">1e-3</span><span class="p">):</span>
-    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Fractionally difference time series via GPU.</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Fractionally difference time series via GPU.</span>
 
 <span class="sd">    Args:</span>
 <span class="sd">        df (pd.DataFrame): dataframe of raw time series values.</span>
@@ -2858,7 +2998,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -2921,12 +3061,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/machine_learning/gpu/rapids_cudf/index.html b/machine_learning/gpu/rapids_cudf/index.html
index 96fdb8cb9..5d0461c66 100644
--- a/machine_learning/gpu/rapids_cudf/index.html
+++ b/machine_learning/gpu/rapids_cudf/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/machine_learning/gpu/rapids_cudf/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -256,6 +256,22 @@
         
   
   
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
     
   
 
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -908,6 +924,130 @@
 
   
   
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
+      
+      <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
     
   
   
@@ -915,14 +1055,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" checked>
       
-      <label class="md-nav__link" for="__nav_4">
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -952,14 +1092,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" checked>
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1099,14 +1239,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1237,14 +1377,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1260,14 +1400,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1320,14 +1460,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -2167,7 +2307,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -2230,12 +2370,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/machine_learning/intro/index.html b/machine_learning/intro/index.html
index 66c3a6d58..69c1b722d 100644
--- a/machine_learning/intro/index.html
+++ b/machine_learning/intro/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/machine_learning/intro/">
       
       <link rel="icon" href="../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -256,6 +256,22 @@
         
   
   
+
+
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
     
   
 
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -908,6 +924,130 @@
 
   
   
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
+      
+      <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
     
   
   
@@ -915,14 +1055,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" checked>
       
-      <label class="md-nav__link" for="__nav_4">
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -960,14 +1100,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1020,14 +1160,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1158,14 +1298,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1181,14 +1321,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1241,14 +1381,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1452,7 +1592,7 @@
                 
                 <h1 id="machine-learning">Machine Learning<a class="headerlink" href="#machine-learning" title="Permanent link">&para;</a></h1>
 <p>We'll be covering both CPU and GPU implementations of machine learning algorithms and data wrangling libraries. </p>
-<div class="tabbed-set" data-tabs="1:1"><input checked="checked" id="__tabbed_1_1" name="__tabbed_1" type="radio" /><label for="__tabbed_1_1">Packages and Languages you will Learn to Use</label><div class="tabbed-content">
+<div class="tabbed-set" data-tabs="1:1"><input checked="checked" id="machine-learning-packages-and-languages-you-will-learn-to-use" name="__tabbed_1" type="radio" /><label for="machine-learning-packages-and-languages-you-will-learn-to-use">Packages and Languages you will Learn to Use</label><div class="tabbed-content">
 <ul>
 <li>Python</li>
 <li>RAPIDS cuDF</li>
@@ -1496,7 +1636,7 @@ <h1 id="machine-learning">Machine Learning<a class="headerlink" href="#machine-l
     <nav class="md-footer__inner md-grid" aria-label="Footer">
       
         
-        <a href="../../deep_learning/production_pytorch/speed_optimization_basics_numba/" class="md-footer__link md-footer__link--prev" aria-label="Previous: Speed Optimization Basics Numba" rel="prev">
+        <a href="../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-footer__link md-footer__link--prev" aria-label="Previous: LLMs Introduction and Hyperparameter Tuning" rel="prev">
           <div class="md-footer__button md-icon">
             <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12z"/></svg>
           </div>
@@ -1505,7 +1645,7 @@ <h1 id="machine-learning">Machine Learning<a class="headerlink" href="#machine-l
               <span class="md-footer__direction">
                 Previous
               </span>
-              Speed Optimization Basics Numba
+              LLMs Introduction and Hyperparameter Tuning
             </div>
           </div>
         </a>
@@ -1533,7 +1673,7 @@ <h1 id="machine-learning">Machine Learning<a class="headerlink" href="#machine-l
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1596,12 +1736,16 @@ <h1 id="machine-learning">Machine Learning<a class="headerlink" href="#machine-l
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/news/ammi_facebook_google_recap_2018_11_21/index.html b/news/ammi_facebook_google_recap_2018_11_21/index.html
index f85e23e57..934e7bf44 100644
--- a/news/ammi_facebook_google_recap_2018_11_21/index.html
+++ b/news/ammi_facebook_google_recap_2018_11_21/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/news/ammi_facebook_google_recap_2018_11_21/">
       
       <link rel="icon" href="../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1008,14 +1148,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1146,14 +1286,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1169,14 +1309,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1231,14 +1371,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" checked>
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1572,7 +1712,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1635,12 +1775,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/news/dbs_gpu_rapids_nvidia_ensemble_frac_diff/index.html b/news/dbs_gpu_rapids_nvidia_ensemble_frac_diff/index.html
index d93e70c0b..aa3b53112 100644
--- a/news/dbs_gpu_rapids_nvidia_ensemble_frac_diff/index.html
+++ b/news/dbs_gpu_rapids_nvidia_ensemble_frac_diff/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/news/dbs_gpu_rapids_nvidia_ensemble_frac_diff/">
       
       <link rel="icon" href="../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1008,14 +1148,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1146,14 +1286,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1169,14 +1309,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1231,14 +1371,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" checked>
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1569,7 +1709,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1632,12 +1772,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/news/deep_learning_wizard_1y_2018_06_01/index.html b/news/deep_learning_wizard_1y_2018_06_01/index.html
index 823340fb2..28ba20536 100644
--- a/news/deep_learning_wizard_1y_2018_06_01/index.html
+++ b/news/deep_learning_wizard_1y_2018_06_01/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/news/deep_learning_wizard_1y_2018_06_01/">
       
       <link rel="icon" href="../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1008,14 +1148,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1146,14 +1286,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1169,14 +1309,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1231,14 +1371,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" checked>
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1497,11 +1637,11 @@
                 
                 <h1 id="featured-on-pytorch-website">Featured on PyTorch Website<a class="headerlink" href="#featured-on-pytorch-website" title="Permanent link">&para;</a></h1>
 <h2 id="pytorch-a-year-later">PyTorch a Year Later<a class="headerlink" href="#pytorch-a-year-later" title="Permanent link">&para;</a></h2>
-<p>We are featured on <a href="https://pytorch.org/2018/01/19/a-year-in.html">PyTorch website's post</a> <img alt="👻" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/1f47b.svg" title=":ghost:" /></p>
+<p>We are featured on <a href="https://pytorch.org/2018/01/19/a-year-in.html">PyTorch website's post</a> <img alt="👻" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f47b.svg" title=":ghost:" /></p>
 <p>I used PyTorch from day 1 and I fell in love with it. One year later, PyTorch's community has grown tremendously. I thank Facebook AI Research (FAIR) and everyone who contributed to building one of the best deep learning framework. In particular Soumith Chintala from FAIR where he is continually fixing bugs and making PyTorch better everyday.</p>
 <p>A year later, Deep Learning Wizard has taught more than 2000 students across 100+ countries, and we continue to strive to release new and updated content to enable anyone to build customized deep learning systems to solve many problems. And I'm trying to push for PyTorch workshops at NVIDIA Deep Learning Institute in Singapore and across the world, so stay tuned!</p>
 <p>A big shoutout for <a href="https://www.linkedin.com/in/alfredocanziani/">Alfredo Canziani</a> who's currently a postdoc at NYU co-teaching with Yann LeCun on deep learning classes and he's releasing another technical course on PyTorch that is going to be awesome.</p>
-<p>To more great years ahead for PyTorch <img alt="👏" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/1f44f.svg" title=":clap:" /></p>
+<p>To more great years ahead for PyTorch <img alt="👏" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f44f.svg" title=":clap:" /></p>
 <p>Cheers,
 <br /><a href="https://www.ritchieng.com/">Ritchie Ng</a></p>
                 
@@ -1569,7 +1709,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1632,12 +1772,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/news/deep_learning_wizard_nvidia_inception_2018_05_01/index.html b/news/deep_learning_wizard_nvidia_inception_2018_05_01/index.html
index c7346c27e..3f1bcf118 100644
--- a/news/deep_learning_wizard_nvidia_inception_2018_05_01/index.html
+++ b/news/deep_learning_wizard_nvidia_inception_2018_05_01/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/news/deep_learning_wizard_nvidia_inception_2018_05_01/">
       
       <link rel="icon" href="../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1008,14 +1148,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1146,14 +1286,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1169,14 +1309,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1231,14 +1371,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" checked>
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1555,7 +1695,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1618,12 +1758,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/news/defence_and_science_technology_agency_dsta_nvidia_talk_2016_06/index.html b/news/defence_and_science_technology_agency_dsta_nvidia_talk_2016_06/index.html
index be7e291e2..addd7367c 100644
--- a/news/defence_and_science_technology_agency_dsta_nvidia_talk_2016_06/index.html
+++ b/news/defence_and_science_technology_agency_dsta_nvidia_talk_2016_06/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/news/defence_and_science_technology_agency_dsta_nvidia_talk_2016_06/">
       
       <link rel="icon" href="../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1008,14 +1148,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1146,14 +1286,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1169,14 +1309,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1231,14 +1371,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" checked>
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1570,7 +1710,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1633,12 +1773,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/news/detect_waterbone_debris_ai_for_social_good_icml_2019_06/index.html b/news/detect_waterbone_debris_ai_for_social_good_icml_2019_06/index.html
index 14350adf0..a8d9c75f9 100644
--- a/news/detect_waterbone_debris_ai_for_social_good_icml_2019_06/index.html
+++ b/news/detect_waterbone_debris_ai_for_social_good_icml_2019_06/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/news/detect_waterbone_debris_ai_for_social_good_icml_2019_06/">
       
       <link rel="icon" href="../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1008,14 +1148,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1146,14 +1286,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1169,14 +1309,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1231,14 +1371,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" checked>
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1567,7 +1707,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1630,12 +1770,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/news/facebook_pytorch_devcon_recap_2018_10_02/index.html b/news/facebook_pytorch_devcon_recap_2018_10_02/index.html
index c508ba0fe..7cac5a100 100644
--- a/news/facebook_pytorch_devcon_recap_2018_10_02/index.html
+++ b/news/facebook_pytorch_devcon_recap_2018_10_02/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/news/facebook_pytorch_devcon_recap_2018_10_02/">
       
       <link rel="icon" href="../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1008,14 +1148,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1146,14 +1286,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1169,14 +1309,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1231,14 +1371,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" checked>
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1570,7 +1710,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1633,12 +1773,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/news/facebook_pytorch_developer_conference_2018_09_05/index.html b/news/facebook_pytorch_developer_conference_2018_09_05/index.html
index 18e10af1b..34d6ba0b2 100644
--- a/news/facebook_pytorch_developer_conference_2018_09_05/index.html
+++ b/news/facebook_pytorch_developer_conference_2018_09_05/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/news/facebook_pytorch_developer_conference_2018_09_05/">
       
       <link rel="icon" href="../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1008,14 +1148,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1146,14 +1286,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1169,14 +1309,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1231,14 +1371,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" checked>
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1567,7 +1707,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1630,12 +1770,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/news/it_youth_leader_2019_03_11/index.html b/news/it_youth_leader_2019_03_11/index.html
index 02443f909..b59a728be 100644
--- a/news/it_youth_leader_2019_03_11/index.html
+++ b/news/it_youth_leader_2019_03_11/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/news/it_youth_leader_2019_03_11/">
       
       <link rel="icon" href="../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1008,14 +1148,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1146,14 +1286,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1169,14 +1309,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1231,14 +1371,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" checked>
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1578,7 +1718,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1641,12 +1781,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/news/nanjing_next_nus_tsinghua_ai_finance_healthcare_2018_11_01/index.html b/news/nanjing_next_nus_tsinghua_ai_finance_healthcare_2018_11_01/index.html
index f65d4b555..284d4b006 100644
--- a/news/nanjing_next_nus_tsinghua_ai_finance_healthcare_2018_11_01/index.html
+++ b/news/nanjing_next_nus_tsinghua_ai_finance_healthcare_2018_11_01/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/news/nanjing_next_nus_tsinghua_ai_finance_healthcare_2018_11_01/">
       
       <link rel="icon" href="../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1008,14 +1148,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1146,14 +1286,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1169,14 +1309,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1231,14 +1371,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" checked>
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1569,7 +1709,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1632,12 +1772,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/news/news/index.html b/news/news/index.html
index 01b87c26b..fb06cc3ce 100644
--- a/news/news/index.html
+++ b/news/news/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/news/news/">
       
       <link rel="icon" href="../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1008,14 +1148,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1146,14 +1286,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1169,14 +1309,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1231,14 +1371,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" checked>
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1496,7 +1636,7 @@
                 
                 
                 <h1 id="welcome-to-our-blog">Welcome to our Blog<a class="headerlink" href="#welcome-to-our-blog" title="Permanent link">&para;</a></h1>
-<p>Here, we post news related to Deep Learning Wizard's releases, features and achievements <img alt="👻" class="twemoji" src="https://twemoji.maxcdn.com/v/latest/svg/1f47b.svg" title=":ghost:" /></p>
+<p>Here, we post news related to Deep Learning Wizard's releases, features and achievements <img alt="👻" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f47b.svg" title=":ghost:" /></p>
 <h2 id="notable-news">Notable News<a class="headerlink" href="#notable-news" title="Permanent link">&para;</a></h2>
 <ul class="task-list">
 <li class="task-list-item"><label class="task-list-control"><input type="checkbox" disabled checked/><span class="task-list-indicator"></span></label> Oral Presentation for AI for Social Good Workshop ICML, Long Beach, Los Angeles, USA, June 2019</li>
@@ -1573,7 +1713,7 @@ <h2 id="notable-news">Notable News<a class="headerlink" href="#notable-news" tit
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1636,12 +1776,16 @@ <h2 id="notable-news">Notable News<a class="headerlink" href="#notable-news" tit
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/news/nvidia_nus_mit_datathon_2018_07_05/index.html b/news/nvidia_nus_mit_datathon_2018_07_05/index.html
index 616923d77..37aed72f9 100644
--- a/news/nvidia_nus_mit_datathon_2018_07_05/index.html
+++ b/news/nvidia_nus_mit_datathon_2018_07_05/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/news/nvidia_nus_mit_datathon_2018_07_05/">
       
       <link rel="icon" href="../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1008,14 +1148,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1146,14 +1286,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1169,14 +1309,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1231,14 +1371,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" checked>
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1565,7 +1705,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1628,12 +1768,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/news/nvidia_self_driving_cars_talk_2017_06_21/index.html b/news/nvidia_self_driving_cars_talk_2017_06_21/index.html
index 8dac429a3..d9f085a00 100644
--- a/news/nvidia_self_driving_cars_talk_2017_06_21/index.html
+++ b/news/nvidia_self_driving_cars_talk_2017_06_21/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/news/nvidia_self_driving_cars_talk_2017_06_21/">
       
       <link rel="icon" href="../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1008,14 +1148,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1146,14 +1286,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1169,14 +1309,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1231,14 +1371,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" checked>
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1570,7 +1710,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1633,12 +1773,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/ood/intro/index.html b/ood/intro/index.html
index 5fd4ba42c..87019c851 100644
--- a/ood/intro/index.html
+++ b/ood/intro/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/ood/intro/">
       
       <link rel="icon" href="../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -355,7 +371,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -914,11 +930,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -946,14 +1086,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1006,14 +1146,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1144,14 +1284,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1167,14 +1307,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1227,14 +1367,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1439,7 +1579,7 @@
                 <h1 id="out-of-distribution-data">Out-of-distribution Data<a class="headerlink" href="#out-of-distribution-data" title="Permanent link">&para;</a></h1>
 <p>This is a persistent and critical production issue present in any machine learning and deep learning systems, no matter how good the models were trained.</p>
 <p>A single out-of-distribution sample data fed into a well-trained model can result in a few problematic outcomes, and in production-level systems they become fat-tailed critical risks that have outsized consequences.</p>
-<div class="tabbed-set" data-tabs="1:1"><input checked="checked" id="__tabbed_1_1" name="__tabbed_1" type="radio" /><label for="__tabbed_1_1">Problematic consequences of out-of-distribution data</label><div class="tabbed-content">
+<div class="tabbed-set" data-tabs="1:1"><input checked="checked" id="out-of-distribution-data-problematic-consequences-of-out-of-distribution-data" name="__tabbed_1" type="radio" /><label for="out-of-distribution-data-problematic-consequences-of-out-of-distribution-data">Problematic consequences of out-of-distribution data</label><div class="tabbed-content">
 <ol>
 <li><strong>False positive &amp; False Negative</strong>: prediction does not match the ground truth.</li>
 <li><strong>Unreliable True positive &amp; True Negative</strong>: prediction matches the ground truth but due to samples being out-of-distribution, the performance of these predictions becomes very erratic. </li>
@@ -1476,7 +1616,7 @@ <h1 id="out-of-distribution-data">Out-of-distribution Data<a class="headerlink"
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1539,12 +1679,16 @@ <h1 id="out-of-distribution-data">Out-of-distribution Data<a class="headerlink"
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/pipeline/index.html b/pipeline/index.html
index cbcd8e138..b4180b0c1 100644
--- a/pipeline/index.html
+++ b/pipeline/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/pipeline/">
       
       <link rel="icon" href="../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -355,7 +371,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -914,11 +930,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -946,14 +1086,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1006,14 +1146,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1144,14 +1284,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1167,14 +1307,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1227,14 +1367,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1471,7 +1611,7 @@ <h1 id="cpu-to-gpu-production-level-pipeline-for-ai">CPU to GPU Production-level
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1534,12 +1674,16 @@ <h1 id="cpu-to-gpu-production-level-pipeline-for-ai">CPU to GPU Production-level
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/programming/bash/bash/index.html b/programming/bash/bash/index.html
index deae008a3..55eac2432 100644
--- a/programming/bash/bash/index.html
+++ b/programming/bash/bash/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/programming/bash/bash/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1010,14 +1150,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" checked>
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1680,14 +1820,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1703,14 +1843,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1763,14 +1903,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -2486,36 +2626,36 @@ <h1 id="bash">Bash<a class="headerlink" href="#bash" title="Permanent link">&par
 </div>
 <h2 id="creating-and-running-a-bash-file">Creating and Running a Bash File<a class="headerlink" href="#creating-and-running-a-bash-file" title="Permanent link">&para;</a></h2>
 <h3 id="create-bash-file">Create bash file<a class="headerlink" href="#create-bash-file" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code>touch hello_world.sh
+<div class="highlight"><pre><span></span><code>touch<span class="w"> </span>hello_world.sh
 </code></pre></div>
 <h3 id="edit-bash-file-with-hello-world">Edit bash file with Hello World<a class="headerlink" href="#edit-bash-file-with-hello-world" title="Permanent link">&para;</a></h3>
 <p>You can edit with anything like Vim, Sublime, etc.</p>
-<div class="highlight"><pre><span></span><code><span class="nb">echo</span> Hello World!
+<div class="highlight"><pre><span></span><code><span class="nb">echo</span><span class="w"> </span>Hello<span class="w"> </span>World!
 </code></pre></div>
 <h3 id="run-bash-file">Run bash file<a class="headerlink" href="#run-bash-file" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code>bash hello_world.sh
+<div class="highlight"><pre><span></span><code>bash<span class="w"> </span>hello_world.sh
 </code></pre></div>
 <p>This will print out, in your bash:
-<div class="highlight"><pre><span></span><code>Hello World!
+<div class="highlight"><pre><span></span><code>Hello<span class="w"> </span>World!
 </code></pre></div></p>
 <h2 id="calculating">Calculating<a class="headerlink" href="#calculating" title="Permanent link">&para;</a></h2>
 <div class="highlight"><pre><span></span><code><span class="c1"># Add</span>
-<span class="nb">echo</span> <span class="k">$((</span><span class="m">10</span><span class="o">+</span><span class="m">10</span><span class="k">))</span>
+<span class="nb">echo</span><span class="w"> </span><span class="k">$((</span><span class="m">10</span><span class="o">+</span><span class="m">10</span><span class="k">))</span>
 
 <span class="c1"># Subtract</span>
-<span class="nb">echo</span> <span class="k">$((</span><span class="m">10</span><span class="o">-</span><span class="m">10</span><span class="k">))</span>
+<span class="nb">echo</span><span class="w"> </span><span class="k">$((</span><span class="m">10</span><span class="o">-</span><span class="m">10</span><span class="k">))</span>
 
 <span class="c1"># Divide</span>
-<span class="nb">echo</span> <span class="k">$((</span><span class="m">10</span><span class="o">/</span><span class="m">10</span><span class="k">))</span>
+<span class="nb">echo</span><span class="w"> </span><span class="k">$((</span><span class="m">10</span><span class="o">/</span><span class="m">10</span><span class="k">))</span>
 
 <span class="c1"># Multiple</span>
-<span class="nb">echo</span> <span class="k">$((</span><span class="m">10</span><span class="o">*</span><span class="m">10</span><span class="k">))</span>
+<span class="nb">echo</span><span class="w"> </span><span class="k">$((</span><span class="m">10</span><span class="o">*</span><span class="m">10</span><span class="k">))</span>
 
 <span class="c1"># Modulo</span>
-<span class="nb">echo</span> <span class="k">$((</span><span class="m">10</span><span class="o">%</span><span class="m">10</span><span class="k">))</span>
+<span class="nb">echo</span><span class="w"> </span><span class="k">$((</span><span class="m">10</span><span class="o">%</span><span class="m">10</span><span class="k">))</span>
 
 <span class="c1"># Multiple Operations: Divide and Add</span>
-<span class="nb">echo</span> <span class="k">$((</span><span class="m">10</span><span class="o">/</span><span class="m">10</span> <span class="o">+</span> <span class="m">10</span><span class="k">))</span>
+<span class="nb">echo</span><span class="w"> </span><span class="k">$((</span><span class="m">10</span><span class="o">/</span><span class="m">10</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="m">10</span><span class="k">))</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code><span class="m">20</span>
 <span class="m">0</span>
@@ -2526,10 +2666,10 @@ <h2 id="calculating">Calculating<a class="headerlink" href="#calculating" title=
 </code></pre></div>
 <h2 id="loops-and-conditional">Loops and Conditional<a class="headerlink" href="#loops-and-conditional" title="Permanent link">&para;</a></h2>
 <h3 id="for-loop">For Loop<a class="headerlink" href="#for-loop" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code><span class="k">for</span> i <span class="k">in</span> <span class="s1">&#39;A&#39;</span> <span class="s1">&#39;B&#39;</span> <span class="s1">&#39;C&#39;</span>
-    <span class="k">do</span>
-        <span class="nb">echo</span> <span class="nv">$i</span>
-    <span class="k">done</span>
+<div class="highlight"><pre><span></span><code><span class="k">for</span><span class="w"> </span>i<span class="w"> </span><span class="k">in</span><span class="w"> </span><span class="s1">&#39;A&#39;</span><span class="w"> </span><span class="s1">&#39;B&#39;</span><span class="w"> </span><span class="s1">&#39;C&#39;</span>
+<span class="w">    </span><span class="k">do</span>
+<span class="w">        </span><span class="nb">echo</span><span class="w"> </span><span class="nv">$i</span>
+<span class="w">    </span><span class="k">done</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>A
 B
@@ -2537,10 +2677,10 @@ <h3 id="for-loop">For Loop<a class="headerlink" href="#for-loop" title="Permanen
 </code></pre></div>
 <h2 id="for-loop-with-range">For Loop With Range<a class="headerlink" href="#for-loop-with-range" title="Permanent link">&para;</a></h2>
 <p>This will echo the digits 0 to 10 without explicitly requiring to define the whole range of numbers/alphabets like above.</p>
-<div class="highlight"><pre><span></span><code><span class="k">for</span> <span class="o">((</span><span class="nv">i</span><span class="o">=</span><span class="m">0</span><span class="p">;</span> i&lt;<span class="o">=</span><span class="m">10</span><span class="p">;</span> i++<span class="o">))</span><span class="p">;</span>
-    <span class="k">do</span>
-        <span class="nb">echo</span> <span class="nv">$i</span>
-    <span class="k">done</span>
+<div class="highlight"><pre><span></span><code><span class="k">for</span><span class="w"> </span><span class="o">((</span><span class="nv">i</span><span class="o">=</span><span class="m">0</span><span class="p">;</span><span class="w"> </span>i&lt;<span class="o">=</span><span class="m">10</span><span class="p">;</span><span class="w"> </span>i++<span class="o">))</span><span class="p">;</span>
+<span class="w">    </span><span class="k">do</span>
+<span class="w">        </span><span class="nb">echo</span><span class="w"> </span><span class="nv">$i</span>
+<span class="w">    </span><span class="k">done</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code><span class="m">0</span>
 <span class="m">1</span>
@@ -2556,35 +2696,35 @@ <h2 id="for-loop-with-range">For Loop With Range<a class="headerlink" href="#for
 </code></pre></div>
 <h3 id="if-else-conditional">If Else Conditional<a class="headerlink" href="#if-else-conditional" title="Permanent link">&para;</a></h3>
 <p>This is a simple if-else to check if the day of the week is 5, meaning if it is a Friday.</p>
-<div class="highlight"><pre><span></span><code><span class="nv">day</span><span class="o">=</span><span class="k">$(</span>date +%u<span class="k">)</span>
+<div class="highlight"><pre><span></span><code><span class="nv">day</span><span class="o">=</span><span class="k">$(</span>date<span class="w"> </span>+%u<span class="k">)</span>
 
-<span class="k">if</span> <span class="o">[</span> <span class="nv">$day</span> <span class="o">==</span> <span class="m">5</span> <span class="o">]</span><span class="p">;</span>
-    <span class="k">then</span>
-        <span class="nb">echo</span> <span class="s2">&quot;Friday is here!&quot;</span>
+<span class="k">if</span><span class="w"> </span><span class="o">[</span><span class="w"> </span><span class="nv">$day</span><span class="w"> </span><span class="o">==</span><span class="w"> </span><span class="m">5</span><span class="w"> </span><span class="o">]</span><span class="p">;</span>
+<span class="w">    </span><span class="k">then</span>
+<span class="w">        </span><span class="nb">echo</span><span class="w"> </span><span class="s2">&quot;Friday is here!&quot;</span>
 
-    <span class="k">else</span>
-        <span class="nb">echo</span> <span class="s2">&quot;Friday is not here :(&quot;</span>
-        <span class="nb">echo</span> <span class="s2">&quot;Today is day </span><span class="nv">$day</span><span class="s2"> of the week&quot;</span>
-    <span class="k">fi</span>
+<span class="w">    </span><span class="k">else</span>
+<span class="w">        </span><span class="nb">echo</span><span class="w"> </span><span class="s2">&quot;Friday is not here :(&quot;</span>
+<span class="w">        </span><span class="nb">echo</span><span class="w"> </span><span class="s2">&quot;Today is day </span><span class="nv">$day</span><span class="s2"> of the week&quot;</span>
+<span class="w">    </span><span class="k">fi</span>
 </code></pre></div>
 <h3 id="sequentially-running-of-python-scripts">Sequentially Running of Python Scripts<a class="headerlink" href="#sequentially-running-of-python-scripts" title="Permanent link">&para;</a></h3>
 <p>This snippet allows you to to run 3 python scripts sequentially, waiting for each to finish before proceeding to the next.</p>
-<div class="highlight"><pre><span></span><code>python script_1.py
+<div class="highlight"><pre><span></span><code>python<span class="w"> </span>script_1.py
 <span class="nb">wait</span>
 
-python script_2.py 
+python<span class="w"> </span>script_2.py<span class="w"> </span>
 <span class="nb">wait</span>
 
-python script_3.py
+python<span class="w"> </span>script_3.py
 <span class="nb">wait</span>
 
-<span class="nb">echo</span> <span class="s2">&quot;Finished running all 3 scripts in sequence!&quot;</span>
+<span class="nb">echo</span><span class="w"> </span><span class="s2">&quot;Finished running all 3 scripts in sequence!&quot;</span>
 </code></pre></div>
 <h3 id="parallel-running-of-python-scripts">Parallel Running of Python Scripts<a class="headerlink" href="#parallel-running-of-python-scripts" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code>python script_1.py <span class="o">&amp;&amp;</span> script_2.py <span class="o">&amp;&amp;</span> script_3.py
+<div class="highlight"><pre><span></span><code>python<span class="w"> </span>script_1.py<span class="w"> </span><span class="o">&amp;&amp;</span><span class="w"> </span>script_2.py<span class="w"> </span><span class="o">&amp;&amp;</span><span class="w"> </span>script_3.py
 <span class="nb">wait</span>
 
-<span class="nb">echo</span> <span class="s2">&quot;Finished running all 3 scripts in parallel in sequence&quot;</span>
+<span class="nb">echo</span><span class="w"> </span><span class="s2">&quot;Finished running all 3 scripts in parallel in sequence&quot;</span>
 </code></pre></div>
 <h2 id="reading-and-writing-operations">Reading and Writing Operations<a class="headerlink" href="#reading-and-writing-operations" title="Permanent link">&para;</a></h2>
 <h3 id="reading-logs-and-texts">Reading logs and texts<a class="headerlink" href="#reading-logs-and-texts" title="Permanent link">&para;</a></h3>
@@ -2601,35 +2741,35 @@ <h3 id="reading-logs-and-texts">Reading logs and texts<a class="headerlink" href
 Row 10
 </code></pre></div>
 <p>Then run the following command to read it in bash then print it.</p>
-<div class="highlight"><pre><span></span><code><span class="nv">text_file</span><span class="o">=</span><span class="k">$(</span>cat random_text.txt<span class="k">)</span>
-<span class="nb">echo</span> <span class="nv">$text_file</span>
+<div class="highlight"><pre><span></span><code><span class="nv">text_file</span><span class="o">=</span><span class="k">$(</span>cat<span class="w"> </span>random_text.txt<span class="k">)</span>
+<span class="nb">echo</span><span class="w"> </span><span class="nv">$text_file</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Row 1 Row 2 Row 3 Row 4 Row 5 Row 6 Row 7 Row 8 Row 9 Row 10
 </code></pre></div>
 <h2 id="date-operations">Date Operations<a class="headerlink" href="#date-operations" title="Permanent link">&para;</a></h2>
 <h3 id="getting-current-date">Getting Current Date<a class="headerlink" href="#getting-current-date" title="Permanent link">&para;</a></h3>
 <p>This will return the date in the format YYYY-MM-DD for example 2019-06-03.</p>
-<div class="highlight"><pre><span></span><code><span class="nv">DATE</span><span class="o">=</span><span class="sb">`</span>date +%Y-%m-%d<span class="sb">`</span>
-<span class="nb">echo</span> <span class="nv">$DATE</span>
+<div class="highlight"><pre><span></span><code><span class="nv">DATE</span><span class="o">=</span><span class="sb">`</span>date<span class="w"> </span>+%Y-%m-%d<span class="sb">`</span>
+<span class="nb">echo</span><span class="w"> </span><span class="nv">$DATE</span>
 </code></pre></div>
 <h3 id="getting-current-day-of-week">Getting Current Day of Week<a class="headerlink" href="#getting-current-day-of-week" title="Permanent link">&para;</a></h3>
 <p>This will return 1, 2, 3, 4, 5, 6, 7 depending on the day of the week.</p>
-<div class="highlight"><pre><span></span><code><span class="nv">DAY</span><span class="o">=</span><span class="k">$(</span>date +%u<span class="k">)</span>
-<span class="nb">echo</span> <span class="nv">$DAY</span>
+<div class="highlight"><pre><span></span><code><span class="nv">DAY</span><span class="o">=</span><span class="k">$(</span>date<span class="w"> </span>+%u<span class="k">)</span>
+<span class="nb">echo</span><span class="w"> </span><span class="nv">$DAY</span>
 </code></pre></div>
 <h3 id="changing-system-dates-by-1-day">Changing System Dates By 1 Day<a class="headerlink" href="#changing-system-dates-by-1-day" title="Permanent link">&para;</a></h3>
 <p>You can change system dates based on this. Surprisingly, you'll find it useful for testing an environment for deployments in the next day and then shifting it back to the actual day.</p>
-<div class="highlight"><pre><span></span><code>sudo date -s <span class="s1">&#39;next day&#39;</span>
-sudo date -s <span class="s1">&#39;yesterday&#39;</span>
+<div class="highlight"><pre><span></span><code>sudo<span class="w"> </span>date<span class="w"> </span>-s<span class="w"> </span><span class="s1">&#39;next day&#39;</span>
+sudo<span class="w"> </span>date<span class="w"> </span>-s<span class="w"> </span><span class="s1">&#39;yesterday&#39;</span>
 </code></pre></div>
 <p>If you are running some tests via bash and want to disable typing in password you can edit the sudoer file via <code>sudo visudo</code> and adding the following line. Only use <code>sudo visudo</code> and nothing else, as they've a special syntax.</p>
-<div class="highlight"><pre><span></span><code>&lt;username&gt; <span class="nv">ALL</span><span class="o">=(</span>ALL<span class="o">)</span> NOPASSWD: /bin/date
+<div class="highlight"><pre><span></span><code>&lt;username&gt;<span class="w"> </span><span class="nv">ALL</span><span class="o">=(</span>ALL<span class="o">)</span><span class="w"> </span>NOPASSWD:<span class="w"> </span>/bin/date
 </code></pre></div>
 <p>To find out your username, simply just run the command <code>whoami</code>.</p>
 <h2 id="jupyter-utility-commands">Jupyter Utility Commands<a class="headerlink" href="#jupyter-utility-commands" title="Permanent link">&para;</a></h2>
 <h3 id="convert-notebook-to-htmlmarkdown">Convert Notebook to HTML/Markdown<a class="headerlink" href="#convert-notebook-to-htmlmarkdown" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code>jupyter nbconvert --to markdown python.ipynb
-jupyter nbconvert --to html python.ipynb
+<div class="highlight"><pre><span></span><code>jupyter<span class="w"> </span>nbconvert<span class="w"> </span>--to<span class="w"> </span>markdown<span class="w"> </span>python.ipynb
+jupyter<span class="w"> </span>nbconvert<span class="w"> </span>--to<span class="w"> </span>html<span class="w"> </span>python.ipynb
 </code></pre></div>
 <h2 id="bash-convenient-commands">Bash Convenient Commands<a class="headerlink" href="#bash-convenient-commands" title="Permanent link">&para;</a></h2>
 <h3 id="list-directories-only">List directories only<a class="headerlink" href="#list-directories-only" title="Permanent link">&para;</a></h3>
@@ -2663,31 +2803,31 @@ <h3 id="kill-all-python-processes">Kill all python processes<a class="headerlink
 <h3 id="install-deb-files">Install .deb files<a class="headerlink" href="#install-deb-files" title="Permanent link">&para;</a></h3>
 <p><code>sudo apt-get install -f file_name.deb</code></p>
 <h3 id="empty-thrash">Empty thrash<a class="headerlink" href="#empty-thrash" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code>sudo apt-get install trash-cli
+<div class="highlight"><pre><span></span><code>sudo<span class="w"> </span>apt-get<span class="w"> </span>install<span class="w"> </span>trash-cli
 thrash-empty
 </code></pre></div>
 <h3 id="fix-git-permissions">Fix Git Permissions<a class="headerlink" href="#fix-git-permissions" title="Permanent link">&para;</a></h3>
 <p>There are times whether you're on Windows (frequent), Mac, or Linux, you might encounter a permission error although you have obviously set it up correctly and it was last working. This is quick fix you can run. </p>
-<div class="highlight"><pre><span></span><code><span class="nb">eval</span> <span class="sb">`</span>ssh-agent -s<span class="sb">`</span> <span class="o">&amp;&amp;</span> ssh-add ~/.ssh/github
+<div class="highlight"><pre><span></span><code><span class="nb">eval</span><span class="w"> </span><span class="sb">`</span>ssh-agent<span class="w"> </span>-s<span class="sb">`</span><span class="w"> </span><span class="o">&amp;&amp;</span><span class="w"> </span>ssh-add<span class="w"> </span>~/.ssh/github
 </code></pre></div>
 <p>If you want, you can even make an alias <code>fixgit</code> and quickly call it in bash to make it easier to fix whenever you face this issue. It's a quick fix, there're more permnanent ways to fix it if you want. But this is a quick fix section. </p>
 <p>In your <code>.bashrc</code> or <code>.zshrc</code>, include the following:</p>
-<div class="highlight"><pre><span></span><code><span class="nb">alias</span> <span class="nv">fixgit</span><span class="o">=</span><span class="s1">&#39;eval `ssh-agent -s` &amp;&amp; ssh-add ~/.ssh/github&#39;</span>
+<div class="highlight"><pre><span></span><code><span class="nb">alias</span><span class="w"> </span><span class="nv">fixgit</span><span class="o">=</span><span class="s1">&#39;eval `ssh-agent -s` &amp;&amp; ssh-add ~/.ssh/github&#39;</span>
 </code></pre></div>
 <h3 id="get-git-repo-information">Get Git Repo Information<a class="headerlink" href="#get-git-repo-information" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code>curl https://api.github.com/&lt;user&gt;/&lt;repo_name&gt;
+<div class="highlight"><pre><span></span><code>curl<span class="w"> </span>https://api.github.com/&lt;user&gt;/&lt;repo_name&gt;
 </code></pre></div>
 <h2 id="conda-commands">Conda Commands<a class="headerlink" href="#conda-commands" title="Permanent link">&para;</a></h2>
 <h3 id="check-conda-environment">Check conda environment<a class="headerlink" href="#check-conda-environment" title="Permanent link">&para;</a></h3>
 <p><code>conda env list</code></p>
 <h3 id="create-conda-kernel">Create conda kernel<a class="headerlink" href="#create-conda-kernel" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code>conda create -n kernel_name <span class="nv">python</span><span class="o">=</span><span class="m">3</span>.6
-<span class="nb">source</span> activate kernel_name
+<div class="highlight"><pre><span></span><code>conda<span class="w"> </span>create<span class="w"> </span>-n<span class="w"> </span>kernel_name<span class="w"> </span><span class="nv">python</span><span class="o">=</span><span class="m">3</span>.6
+<span class="nb">source</span><span class="w"> </span>activate<span class="w"> </span>kernel_name
 </code></pre></div>
 <h3 id="install-conda-kernel">Install conda kernel<a class="headerlink" href="#install-conda-kernel" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code>conda install ipykernel
-<span class="nb">source</span> activate kernel_name
-python -m ipykernel install --user --name kernel_name --display-name kernel_name
+<div class="highlight"><pre><span></span><code>conda<span class="w"> </span>install<span class="w"> </span>ipykernel
+<span class="nb">source</span><span class="w"> </span>activate<span class="w"> </span>kernel_name
+python<span class="w"> </span>-m<span class="w"> </span>ipykernel<span class="w"> </span>install<span class="w"> </span>--user<span class="w"> </span>--name<span class="w"> </span>kernel_name<span class="w"> </span>--display-name<span class="w"> </span>kernel_name
 </code></pre></div>
 <h3 id="remove-conda-kernel">Remove conda kernel<a class="headerlink" href="#remove-conda-kernel" title="Permanent link">&para;</a></h3>
 <p><code>conda env remove -n kernel_name</code></p>
@@ -2696,24 +2836,24 @@ <h3 id="recovering-problematic-conda-installation">Recovering problematic conda
 <span class="c1"># Link: https://docs.conda.io/en/latest/miniconda.html</span>
 
 <span class="c1"># Backup existing miniconda environment that may have problems</span>
-mv miniconda3 miniconda3_backup
+mv<span class="w"> </span>miniconda3<span class="w"> </span>miniconda3_backup
 
 <span class="c1"># Install miniconda</span>
-bash Miniconda3-latest-Linux-x86_64.sh
+bash<span class="w"> </span>Miniconda3-latest-Linux-x86_64.sh
 
 <span class="c1"># Restore old environment settings</span>
-rsync -a miniconda3_backup/ miniconda3/
+rsync<span class="w"> </span>-a<span class="w"> </span>miniconda3_backup/<span class="w"> </span>miniconda3/
 </code></pre></div>
 <h2 id="internet-operations">Internet Operations<a class="headerlink" href="#internet-operations" title="Permanent link">&para;</a></h2>
 <h3 id="checking-internet-availability">Checking Internet Availability<a class="headerlink" href="#checking-internet-availability" title="Permanent link">&para;</a></h3>
 <p>This script will return whether your internet is fine or not without using pings.</p>
 <p>Pings can often be rendered unusable when the network administrator disables ICMP to prevent the origination of ping floods from the data centre.</p>
-<div class="highlight"><pre><span></span><code><span class="k">if</span> nc -zw1 google.com <span class="m">443</span><span class="p">;</span>
-    <span class="k">then</span>
-        <span class="nb">echo</span> <span class="s2">&quot;INTERNET: OK&quot;</span>
-    <span class="k">else</span>
-        <span class="nb">echo</span> <span class="s2">&quot;INTERNET: NOT OK&quot;</span>
-    <span class="k">fi</span>
+<div class="highlight"><pre><span></span><code><span class="k">if</span><span class="w"> </span>nc<span class="w"> </span>-zw1<span class="w"> </span>google.com<span class="w"> </span><span class="m">443</span><span class="p">;</span>
+<span class="w">    </span><span class="k">then</span>
+<span class="w">        </span><span class="nb">echo</span><span class="w"> </span><span class="s2">&quot;INTERNET: OK&quot;</span>
+<span class="w">    </span><span class="k">else</span>
+<span class="w">        </span><span class="nb">echo</span><span class="w"> </span><span class="s2">&quot;INTERNET: NOT OK&quot;</span>
+<span class="w">    </span><span class="k">fi</span>
 </code></pre></div>
 <h2 id="cron-operations">Cron Operations<a class="headerlink" href="#cron-operations" title="Permanent link">&para;</a></h2>
 <h3 id="edit-cron">Edit Cron<a class="headerlink" href="#edit-cron" title="Permanent link">&para;</a></h3>
@@ -2728,23 +2868,23 @@ <h3 id="edit-cron">Edit Cron<a class="headerlink" href="#edit-cron" title="Perma
  * * * * * command to be executed
 </code></pre></div></p>
 <p>Edit cron with this command:
-<div class="highlight"><pre><span></span><code>sudo crontab -e
+<div class="highlight"><pre><span></span><code>sudo<span class="w"> </span>crontab<span class="w"> </span>-e
 </code></pre></div></p>
 <h3 id="list-cron">List Cron<a class="headerlink" href="#list-cron" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code>sudo crontab -l
+<div class="highlight"><pre><span></span><code>sudo<span class="w"> </span>crontab<span class="w"> </span>-l
 </code></pre></div>
 <h3 id="status-start-stop-and-restart">Status, Start, Stop and Restart<a class="headerlink" href="#status-start-stop-and-restart" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code>sudo service cron status
-sudo service cron stop
-sudo service cron start
-sudo service cron restart
+<div class="highlight"><pre><span></span><code>sudo<span class="w"> </span>service<span class="w"> </span>cron<span class="w"> </span>status
+sudo<span class="w"> </span>service<span class="w"> </span>cron<span class="w"> </span>stop
+sudo<span class="w"> </span>service<span class="w"> </span>cron<span class="w"> </span>start
+sudo<span class="w"> </span>service<span class="w"> </span>cron<span class="w"> </span>restart
 </code></pre></div>
 <h3 id="cron-debugging">Cron Debugging<a class="headerlink" href="#cron-debugging" title="Permanent link">&para;</a></h3>
 <p>Install postfix for local routing of errors (choose local option):
-<div class="highlight"><pre><span></span><code>sudo apt-get install postfix
+<div class="highlight"><pre><span></span><code>sudo<span class="w"> </span>apt-get<span class="w"> </span>install<span class="w"> </span>postfix
 </code></pre></div></p>
 <p>Restart cron to see for any errors posted (if not errors, there will be no file, be patient before errors are posted):
-<div class="highlight"><pre><span></span><code>sudo cat /var/mail/root
+<div class="highlight"><pre><span></span><code>sudo<span class="w"> </span>cat<span class="w"> </span>/var/mail/root
 </code></pre></div></p>
 <h3 id="cron-bash-fix">Cron Bash Fix<a class="headerlink" href="#cron-bash-fix" title="Permanent link">&para;</a></h3>
 <p>Cron uses <code>/bin/sh</code> as the default shell. Typically you would realize you're using <code>/bin/bash</code> as your shell, so this typically needs to be rectified before you can use cron to schedule commands as if it were your usual bash.</p>
@@ -2767,23 +2907,23 @@ <h3 id="cron-running-processes">Cron Running Processes<a class="headerlink" href
 <p>You can get the PID of the cron process and then end it with <code>sudo pkill -s &lt;PID&gt;</code></p>
 <h2 id="hardware-information">Hardware Information<a class="headerlink" href="#hardware-information" title="Permanent link">&para;</a></h2>
 <h3 id="comprehensive-cpu-information">Comprehensive CPU Information<a class="headerlink" href="#comprehensive-cpu-information" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code>cat /proc/cpuinfo
+<div class="highlight"><pre><span></span><code>cat<span class="w"> </span>/proc/cpuinfo
 </code></pre></div>
 <h3 id="number-of-cpu-threads">Number of CPU Threads<a class="headerlink" href="#number-of-cpu-threads" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code>!grep -c ^processor /proc/cpuinfo
+<div class="highlight"><pre><span></span><code>!grep<span class="w"> </span>-c<span class="w"> </span>^processor<span class="w"> </span>/proc/cpuinfo
 </code></pre></div>
 <p>or </p>
 <div class="highlight"><pre><span></span><code>nproc
 </code></pre></div>
 <h3 id="cpu-model-name">CPU Model Name<a class="headerlink" href="#cpu-model-name" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code>!cat /proc/cpuinfo <span class="p">|</span> grep <span class="s2">&quot;model name&quot;</span> 
+<div class="highlight"><pre><span></span><code>!cat<span class="w"> </span>/proc/cpuinfo<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span><span class="s2">&quot;model name&quot;</span><span class="w"> </span>
 </code></pre></div>
 <h3 id="check-available-ram">Check Available RAM<a class="headerlink" href="#check-available-ram" title="Permanent link">&para;</a></h3>
 <h4 id="in-mb">In MB<a class="headerlink" href="#in-mb" title="Permanent link">&para;</a></h4>
-<div class="highlight"><pre><span></span><code>free -m
+<div class="highlight"><pre><span></span><code>free<span class="w"> </span>-m
 </code></pre></div>
 <h4 id="in-gb">In GB<a class="headerlink" href="#in-gb" title="Permanent link">&para;</a></h4>
-<div class="highlight"><pre><span></span><code>free -g
+<div class="highlight"><pre><span></span><code>free<span class="w"> </span>-g
 </code></pre></div>
                 
               
@@ -2850,7 +2990,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -2913,12 +3053,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/programming/cpp/cpp/index.html b/programming/cpp/cpp/index.html
index b3727f1ec..4941a19e6 100644
--- a/programming/cpp/cpp/index.html
+++ b/programming/cpp/cpp/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/programming/cpp/cpp/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1010,14 +1150,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" checked>
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1818,14 +1958,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1841,14 +1981,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1901,14 +2041,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -2762,11 +2902,11 @@ <h1 id="c">C++<a class="headerlink" href="#c" title="Permanent link">&para;</a><
 </div>
 <h2 id="installation-of-interactive-c17">Installation of Interactive C++17<a class="headerlink" href="#installation-of-interactive-c17" title="Permanent link">&para;</a></h2>
 <p>Xeus-Cling is a game-changer where similar to Python Jupyter Notebooks, we can run C++ Jupyter Notebooks now. Run the following bash commands in sequence to create a C++ kernel for Jupyter Notebook.</p>
-<div class="highlight"><pre><span></span><code>conda create -n cpp
-<span class="nb">source</span> activate cpp
-conda install -c conda-forge xeus-cling
-jupyter kernelspec install --user /home/ritchie/miniconda3/envs/cpp/share/jupyter/kernels/xcpp17
-jupyter notebook
+<div class="highlight"><pre><span></span><code>conda<span class="w"> </span>create<span class="w"> </span>-n<span class="w"> </span>cpp
+<span class="nb">source</span><span class="w"> </span>activate<span class="w"> </span>cpp
+conda<span class="w"> </span>install<span class="w"> </span>-c<span class="w"> </span>conda-forge<span class="w"> </span>xeus-cling
+jupyter<span class="w"> </span>kernelspec<span class="w"> </span>install<span class="w"> </span>--user<span class="w"> </span>/home/ritchie/miniconda3/envs/cpp/share/jupyter/kernels/xcpp17
+jupyter<span class="w"> </span>notebook
 </code></pre></div>
 <h2 id="printing">Printing<a class="headerlink" href="#printing" title="Permanent link">&para;</a></h2>
 <h3 id="printing-single-line">Printing Single Line<a class="headerlink" href="#printing-single-line" title="Permanent link">&para;</a></h3>
@@ -2774,30 +2914,30 @@ <h3 id="printing-single-line">Printing Single Line<a class="headerlink" href="#p
 
 <span class="c1">// This is a preprocessor instruction that</span>
 <span class="c1">// essentially places the file with the name iostream into this spot</span>
-<span class="cp">#include</span> <span class="cpf">&lt;iostream&gt;</span><span class="cp"></span>
+<span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;iostream&gt;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code><span class="c1">// Print one line</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Using Xeus Cling&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Using Xeus Cling&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Using Xeus Cling
 </code></pre></div>
 <h3 id="printing-2-lines">Printing 2 Lines<a class="headerlink" href="#printing-2-lines" title="Permanent link">&para;</a></h3>
 <div class="highlight"><pre><span></span><code><span class="c1">// Print two lines</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;First Line </span><span class="se">\n</span><span class="s">Second Line&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;First Line </span><span class="se">\n</span><span class="s">Second Line&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>First Line 
 Second Line
 </code></pre></div>
 <h3 id="printing-with-tabs">Printing with Tabs<a class="headerlink" href="#printing-with-tabs" title="Permanent link">&para;</a></h3>
 <div class="highlight"><pre><span></span><code><span class="c1">// Print numbers with nicely formatted tabs</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;One hundred:</span><span class="se">\t</span><span class="s">&quot;</span><span class="p">;</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="p">(</span><span class="kt">float</span><span class="p">)</span> <span class="mi">1000</span><span class="o">/</span><span class="mi">10</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;One hundred:</span><span class="se">\t</span><span class="s">&quot;</span><span class="p">;</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="p">(</span><span class="kt">float</span><span class="p">)</span><span class="w"> </span><span class="mi">1000</span><span class="o">/</span><span class="mi">10</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Two hundred:</span><span class="se">\t</span><span class="s">&quot;</span><span class="p">;</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="p">(</span><span class="kt">double</span><span class="p">)</span> <span class="mi">2000</span><span class="o">/</span><span class="mi">10</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Two hundred:</span><span class="se">\t</span><span class="s">&quot;</span><span class="p">;</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="p">(</span><span class="kt">double</span><span class="p">)</span><span class="w"> </span><span class="mi">2000</span><span class="o">/</span><span class="mi">10</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Three hundred:</span><span class="se">\t</span><span class="s">&quot;</span><span class="p">;</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="p">(</span><span class="kt">double</span><span class="p">)</span> <span class="mi">3000</span><span class="o">/</span><span class="mi">10</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Three hundred:</span><span class="se">\t</span><span class="s">&quot;</span><span class="p">;</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="p">(</span><span class="kt">double</span><span class="p">)</span><span class="w"> </span><span class="mi">3000</span><span class="o">/</span><span class="mi">10</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>One hundred:    100
 Two hundred:    200
@@ -2807,8 +2947,8 @@ <h3 id="easier-printing-subjective-with-namespaces">Easier Printing (subjective)
 <ul>
 <li>Gets irritating to use <code>std</code> in front of <code>cout</code> and <code>endl</code> to keep printing so we can use namespaces</li>
 </ul>
-<div class="highlight"><pre><span></span><code><span class="k">using</span> <span class="k">namespace</span> <span class="nn">std</span><span class="p">;</span>
-<span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;No need for messy std::&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">endl</span><span class="p">;</span>
+<div class="highlight"><pre><span></span><code><span class="k">using</span><span class="w"> </span><span class="k">namespace</span><span class="w"> </span><span class="nn">std</span><span class="p">;</span>
+<span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;No need for messy std::&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">endl</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>No need for messy std::
 </code></pre></div>
@@ -2905,17 +3045,17 @@ <h3 id="integer-sizes">Integer sizes<a class="headerlink" href="#integer-sizes"
 </li>
 </ul>
 <h4 id="integer-size">Integer size<a class="headerlink" href="#integer-size" title="Permanent link">&para;</a></h4>
-<div class="highlight"><pre><span></span><code><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Size of an integer: &quot;</span> <span class="o">&lt;&lt;</span> <span class="k">sizeof</span> <span class="p">(</span><span class="kt">int</span><span class="p">);</span>
+<div class="highlight"><pre><span></span><code><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Size of an integer: &quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="k">sizeof</span><span class="w"> </span><span class="p">(</span><span class="kt">int</span><span class="p">);</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Size of an integer: 4
 </code></pre></div>
 <h4 id="short-integer-size">Short integer size<a class="headerlink" href="#short-integer-size" title="Permanent link">&para;</a></h4>
-<div class="highlight"><pre><span></span><code><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Size of a short integer: &quot;</span> <span class="o">&lt;&lt;</span> <span class="k">sizeof</span> <span class="p">(</span><span class="kt">short</span> <span class="kt">int</span><span class="p">);</span>
+<div class="highlight"><pre><span></span><code><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Size of a short integer: &quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="k">sizeof</span><span class="w"> </span><span class="p">(</span><span class="kt">short</span><span class="w"> </span><span class="kt">int</span><span class="p">);</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Size of a short integer: 2
 </code></pre></div>
 <h4 id="long-integer-size">Long integer size<a class="headerlink" href="#long-integer-size" title="Permanent link">&para;</a></h4>
-<div class="highlight"><pre><span></span><code><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Size of an long integer: &quot;</span> <span class="o">&lt;&lt;</span> <span class="k">sizeof</span> <span class="p">(</span><span class="kt">long</span> <span class="kt">int</span><span class="p">);</span>
+<div class="highlight"><pre><span></span><code><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Size of an long integer: &quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="k">sizeof</span><span class="w"> </span><span class="p">(</span><span class="kt">long</span><span class="w"> </span><span class="kt">int</span><span class="p">);</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Size of an long integer: 8
 </code></pre></div>
@@ -2925,41 +3065,41 @@ <h3 id="unsigned-or-signed-integers">Unsigned or signed integers<a class="header
 <li>Signed integers: can hold positive/negative integers</li>
 </ul>
 <h4 id="signed-short-integer">Signed short integer<a class="headerlink" href="#signed-short-integer" title="Permanent link">&para;</a></h4>
-<div class="highlight"><pre><span></span><code><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Size of an signed short integer: &quot;</span> <span class="o">&lt;&lt;</span> <span class="k">sizeof</span> <span class="p">(</span><span class="kt">signed</span> <span class="kt">short</span> <span class="kt">int</span><span class="p">);</span>
+<div class="highlight"><pre><span></span><code><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Size of an signed short integer: &quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="k">sizeof</span><span class="w"> </span><span class="p">(</span><span class="kt">signed</span><span class="w"> </span><span class="kt">short</span><span class="w"> </span><span class="kt">int</span><span class="p">);</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Size of an signed short integer: 2
 </code></pre></div>
 <h4 id="unsigned-short-integer">Unsigned short integer<a class="headerlink" href="#unsigned-short-integer" title="Permanent link">&para;</a></h4>
-<div class="highlight"><pre><span></span><code><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Size of an unsigned short integer: &quot;</span> <span class="o">&lt;&lt;</span> <span class="k">sizeof</span> <span class="p">(</span><span class="kt">unsigned</span> <span class="kt">short</span> <span class="kt">int</span><span class="p">);</span>
+<div class="highlight"><pre><span></span><code><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Size of an unsigned short integer: &quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="k">sizeof</span><span class="w"> </span><span class="p">(</span><span class="kt">unsigned</span><span class="w"> </span><span class="kt">short</span><span class="w"> </span><span class="kt">int</span><span class="p">);</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Size of an unsigned short integer: 2
 </code></pre></div>
 <h4 id="signed-long-integer">Signed long integer<a class="headerlink" href="#signed-long-integer" title="Permanent link">&para;</a></h4>
-<div class="highlight"><pre><span></span><code><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Size of an signed long integer: &quot;</span> <span class="o">&lt;&lt;</span> <span class="k">sizeof</span> <span class="p">(</span><span class="kt">signed</span> <span class="kt">long</span> <span class="kt">int</span><span class="p">);</span>
+<div class="highlight"><pre><span></span><code><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Size of an signed long integer: &quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="k">sizeof</span><span class="w"> </span><span class="p">(</span><span class="kt">signed</span><span class="w"> </span><span class="kt">long</span><span class="w"> </span><span class="kt">int</span><span class="p">);</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Size of an signed long integer: 8
 </code></pre></div>
 <h4 id="unsigned-long-integer">Unsigned long integer<a class="headerlink" href="#unsigned-long-integer" title="Permanent link">&para;</a></h4>
-<div class="highlight"><pre><span></span><code><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Size of an unsigned long integer: &quot;</span> <span class="o">&lt;&lt;</span> <span class="k">sizeof</span> <span class="p">(</span><span class="kt">unsigned</span> <span class="kt">long</span> <span class="kt">int</span><span class="p">);</span>
+<div class="highlight"><pre><span></span><code><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Size of an unsigned long integer: &quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="k">sizeof</span><span class="w"> </span><span class="p">(</span><span class="kt">unsigned</span><span class="w"> </span><span class="kt">long</span><span class="w"> </span><span class="kt">int</span><span class="p">);</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Size of an unsigned long integer: 8
 </code></pre></div>
 <h2 id="constants">Constants<a class="headerlink" href="#constants" title="Permanent link">&para;</a></h2>
 <h3 id="literal-constants">Literal Constants<a class="headerlink" href="#literal-constants" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code><span class="kt">int</span> <span class="n">varOne</span> <span class="o">=</span> <span class="mi">20</span><span class="p">;</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="n">varOne</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<div class="highlight"><pre><span></span><code><span class="kt">int</span><span class="w"> </span><span class="n">varOne</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">20</span><span class="p">;</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">varOne</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>20
 </code></pre></div>
 <h3 id="enumerated-constants">Enumerated Constants<a class="headerlink" href="#enumerated-constants" title="Permanent link">&para;</a></h3>
 <p>This enables you to create a new type! In this example we create a new type <code>directions</code> containing <code>Up</code>, <code>Down</code>, <code>Left</code>, and <code>Right</code>.</p>
-<div class="highlight"><pre><span></span><code><span class="k">enum</span> <span class="nc">directions</span> <span class="p">{</span><span class="n">Up</span><span class="p">,</span> <span class="n">Down</span><span class="p">,</span> <span class="n">Left</span><span class="p">,</span> <span class="n">Right</span><span class="p">};</span>
+<div class="highlight"><pre><span></span><code><span class="k">enum</span><span class="w"> </span><span class="nc">directions</span><span class="w"> </span><span class="p">{</span><span class="n">Up</span><span class="p">,</span><span class="w"> </span><span class="n">Down</span><span class="p">,</span><span class="w"> </span><span class="n">Left</span><span class="p">,</span><span class="w"> </span><span class="n">Right</span><span class="p">};</span>
 
-<span class="n">directions</span> <span class="n">goWhere</span><span class="p">;</span>
-<span class="n">goWhere</span> <span class="o">=</span> <span class="n">Right</span><span class="p">;</span>
+<span class="n">directions</span><span class="w"> </span><span class="n">goWhere</span><span class="p">;</span>
+<span class="n">goWhere</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">Right</span><span class="p">;</span>
 
-<span class="k">if</span> <span class="p">(</span><span class="n">goWhere</span> <span class="o">==</span> <span class="n">Right</span><span class="p">)</span>
-    <span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Go right&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="n">goWhere</span><span class="w"> </span><span class="o">==</span><span class="w"> </span><span class="n">Right</span><span class="p">)</span>
+<span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Go right&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Go right
 </code></pre></div>
@@ -2968,21 +3108,21 @@ <h3 id="function-without-return-value">Function Without Return Value<a class="he
 <p>Syntax generally follows <code>void FunctionName(argOne, argTwo)</code> to define the function followed by <code>FuntionName()</code> to call the function.</p>
 <div class="highlight"><pre><span></span><code><span class="c1">// Usage of void when the function does not return anything</span>
 <span class="c1">// In this example, this function prints out the multiplication result of two given numbers</span>
-<span class="kt">void</span> <span class="nf">MultiplyTwoNumbers</span><span class="p">(</span><span class="kt">int</span> <span class="n">firstNum</span><span class="p">,</span> <span class="kt">int</span> <span class="n">secondNum</span><span class="p">)</span>
+<span class="kt">void</span><span class="w"> </span><span class="nf">MultiplyTwoNumbers</span><span class="p">(</span><span class="kt">int</span><span class="w"> </span><span class="n">firstNum</span><span class="p">,</span><span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">secondNum</span><span class="p">)</span>
 <span class="p">{</span>
-    <span class="c1">// Define variable as integer type</span>
-    <span class="kt">long</span> <span class="kt">int</span> <span class="n">value</span><span class="p">;</span>
-    <span class="n">value</span> <span class="o">=</span> <span class="n">firstNum</span> <span class="o">*</span> <span class="n">secondNum</span><span class="p">;</span>
-    <span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="n">value</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span> 
+<span class="w">    </span><span class="c1">// Define variable as integer type</span>
+<span class="w">    </span><span class="kt">long</span><span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">value</span><span class="p">;</span>
+<span class="w">    </span><span class="n">value</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">firstNum</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="n">secondNum</span><span class="p">;</span>
+<span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">value</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span><span class="w"> </span>
 <span class="p">}</span>
 </code></pre></div>
 <h4 id="multiply-two-numbers-3-and-2">Multiply Two Numbers 3 and 2<a class="headerlink" href="#multiply-two-numbers-3-and-2" title="Permanent link">&para;</a></h4>
-<div class="highlight"><pre><span></span><code><span class="n">MultiplyTwoNumbers</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
+<div class="highlight"><pre><span></span><code><span class="n">MultiplyTwoNumbers</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span><span class="w"> </span><span class="mi">2</span><span class="p">)</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>6
 </code></pre></div>
 <h4 id="multiply-two-numbers-6-and-2">Multiply Two Numbers 6 and 2<a class="headerlink" href="#multiply-two-numbers-6-and-2" title="Permanent link">&para;</a></h4>
-<div class="highlight"><pre><span></span><code><span class="n">MultiplyTwoNumbers</span><span class="p">(</span><span class="mi">6</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
+<div class="highlight"><pre><span></span><code><span class="n">MultiplyTwoNumbers</span><span class="p">(</span><span class="mi">6</span><span class="p">,</span><span class="w"> </span><span class="mi">2</span><span class="p">)</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>12
 </code></pre></div>
@@ -3011,23 +3151,23 @@ <h3 id="aliases-function">Aliases Function<a class="headerlink" href="#aliases-f
 </table>
 <div class="highlight"><pre><span></span><code><span class="c1">// Usage of void when the function does not return anything</span>
 <span class="c1">// In this exmaple, this function prints out the multiplication result of two given numbers</span>
-<span class="kt">void</span> <span class="nf">MultiplyTwoNumbersWithAlias</span><span class="p">(</span><span class="kt">int</span> <span class="n">firstNum</span><span class="p">,</span> <span class="kt">int</span> <span class="n">secondNum</span><span class="p">)</span>
+<span class="kt">void</span><span class="w"> </span><span class="nf">MultiplyTwoNumbersWithAlias</span><span class="p">(</span><span class="kt">int</span><span class="w"> </span><span class="n">firstNum</span><span class="p">,</span><span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">secondNum</span><span class="p">)</span>
 <span class="p">{</span>
-    <span class="c1">// Using an alias</span>
-    <span class="k">typedef</span> <span class="kt">unsigned</span> <span class="kt">short</span> <span class="kt">int</span> <span class="n">ushortint</span><span class="p">;</span>
-    <span class="c1">// initializing value variable with ushortint type</span>
-    <span class="n">ushortint</span> <span class="n">value</span><span class="p">;</span>
-    <span class="n">value</span> <span class="o">=</span> <span class="n">firstNum</span> <span class="o">*</span> <span class="n">secondNum</span><span class="p">;</span>
-    <span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="n">value</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span> 
+<span class="w">    </span><span class="c1">// Using an alias</span>
+<span class="w">    </span><span class="k">typedef</span><span class="w"> </span><span class="kt">unsigned</span><span class="w"> </span><span class="kt">short</span><span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">ushortint</span><span class="p">;</span>
+<span class="w">    </span><span class="c1">// initializing value variable with ushortint type</span>
+<span class="w">    </span><span class="n">ushortint</span><span class="w"> </span><span class="n">value</span><span class="p">;</span>
+<span class="w">    </span><span class="n">value</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">firstNum</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="n">secondNum</span><span class="p">;</span>
+<span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">value</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span><span class="w"> </span>
 <span class="p">}</span>
 </code></pre></div>
 <h4 id="multiply-two-numbers-10-and-10">Multiply Two Numbers 10 and 10<a class="headerlink" href="#multiply-two-numbers-10-and-10" title="Permanent link">&para;</a></h4>
-<div class="highlight"><pre><span></span><code><span class="n">MultiplyTwoNumbersWithAlias</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span>
+<div class="highlight"><pre><span></span><code><span class="n">MultiplyTwoNumbersWithAlias</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span><span class="w"> </span><span class="mi">10</span><span class="p">)</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>100
 </code></pre></div>
 <h4 id="multiply-two-numbers-1000-and-65">Multiply Two Numbers 1000 and 65<a class="headerlink" href="#multiply-two-numbers-1000-and-65" title="Permanent link">&para;</a></h4>
-<div class="highlight"><pre><span></span><code><span class="n">MultiplyTwoNumbersWithAlias</span><span class="p">(</span><span class="mi">1000</span><span class="p">,</span> <span class="mi">65</span><span class="p">)</span>
+<div class="highlight"><pre><span></span><code><span class="n">MultiplyTwoNumbersWithAlias</span><span class="p">(</span><span class="mi">1000</span><span class="p">,</span><span class="w"> </span><span class="mi">65</span><span class="p">)</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>65000
 </code></pre></div>
@@ -3036,32 +3176,32 @@ <h4 id="multiply-two-numbers-1000-and-67">Multiply Two Numbers 1000 and 67<a cla
 <li>Notice how you don't get 67,000? This is because our variable <code>value</code> of <code>ushortint</code> type can only hold values up to the integer 65,536.</li>
 <li>What this returns is the remainder of 67,000 - 65,536 = 1464</li>
 </ul>
-<div class="highlight"><pre><span></span><code><span class="n">MultiplyTwoNumbersWithAlias</span><span class="p">(</span><span class="mi">1000</span><span class="p">,</span> <span class="mi">67</span><span class="p">)</span>
+<div class="highlight"><pre><span></span><code><span class="n">MultiplyTwoNumbersWithAlias</span><span class="p">(</span><span class="mi">1000</span><span class="p">,</span><span class="w"> </span><span class="mi">67</span><span class="p">)</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>1464
 </code></pre></div>
-<div class="highlight"><pre><span></span><code><span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="mi">67</span> <span class="o">*</span> <span class="mi">1000</span> <span class="o">-</span> <span class="mi">65536</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<div class="highlight"><pre><span></span><code><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="mi">67</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="mi">1000</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="mi">65536</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>1464
 </code></pre></div>
 <h3 id="function-with-return-value">Function with Return Value<a class="headerlink" href="#function-with-return-value" title="Permanent link">&para;</a></h3>
 <p>Unlike functions without return values where we use <code>void</code> to declare the function, here we use <code>int</code> to declare our function that returns values.</p>
 <div class="highlight"><pre><span></span><code><span class="c1">// In this exmaple, this function returns the value of the multiplication of two numbers</span>
-<span class="kt">int</span> <span class="nf">MultiplyTwoNumbersNoPrint</span><span class="p">(</span><span class="kt">int</span> <span class="n">firstNum</span><span class="p">,</span> <span class="kt">int</span> <span class="n">secondNum</span><span class="p">)</span>
+<span class="kt">int</span><span class="w"> </span><span class="nf">MultiplyTwoNumbersNoPrint</span><span class="p">(</span><span class="kt">int</span><span class="w"> </span><span class="n">firstNum</span><span class="p">,</span><span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">secondNum</span><span class="p">)</span>
 <span class="p">{</span>
-    <span class="c1">// Define variable as integer type</span>
-    <span class="kt">long</span> <span class="kt">int</span> <span class="n">value</span><span class="p">;</span>
-    <span class="n">value</span> <span class="o">=</span> <span class="n">firstNum</span> <span class="o">*</span> <span class="n">secondNum</span><span class="p">;</span>
-    <span class="k">return</span> <span class="n">value</span><span class="p">;</span>
+<span class="w">    </span><span class="c1">// Define variable as integer type</span>
+<span class="w">    </span><span class="kt">long</span><span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">value</span><span class="p">;</span>
+<span class="w">    </span><span class="n">value</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">firstNum</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="n">secondNum</span><span class="p">;</span>
+<span class="w">    </span><span class="k">return</span><span class="w"> </span><span class="n">value</span><span class="p">;</span>
 <span class="p">}</span>
 </code></pre></div>
 <h4 id="call-function">Call Function<a class="headerlink" href="#call-function" title="Permanent link">&para;</a></h4>
 <div class="highlight"><pre><span></span><code><span class="c1">// Declare variable with type</span>
-<span class="kt">long</span> <span class="kt">int</span> <span class="n">returnValue</span><span class="p">;</span>
+<span class="kt">long</span><span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">returnValue</span><span class="p">;</span>
 <span class="c1">// Call function</span>
-<span class="n">returnValue</span> <span class="o">=</span> <span class="n">MultiplyTwoNumbersNoPrint</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">2</span><span class="p">);</span>
+<span class="n">returnValue</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">MultiplyTwoNumbersNoPrint</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span><span class="w"> </span><span class="mi">2</span><span class="p">);</span>
 <span class="c1">// Print variable</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="n">returnValue</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">returnValue</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>20
 </code></pre></div>
@@ -3074,8 +3214,8 @@ <h3 id="function-inner-workings">Function Inner Workings<a class="headerlink" hr
 </li>
 </ul>
 <div class="highlight"><pre><span></span><code><span class="c1">// Code Space</span>
-<span class="kt">int</span> <span class="n">varOneTest</span> <span class="o">=</span> <span class="mi">10</span><span class="p">;</span> <span class="c1">// Instruction pointer 100</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="n">varOneTest</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span> <span class="c1">// Instruction Pointer 102</span>
+<span class="kt">int</span><span class="w"> </span><span class="n">varOneTest</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">10</span><span class="p">;</span><span class="w"> </span><span class="c1">// Instruction pointer 100</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">varOneTest</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span><span class="w"> </span><span class="c1">// Instruction Pointer 102</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>10
 
@@ -3089,26 +3229,26 @@ <h2 id="arrays">Arrays<a class="headerlink" href="#arrays" title="Permanent link
 <p>An array contains a sequence of elements with the same data type. </p>
 <h3 id="creating-an-array">Creating an Array<a class="headerlink" href="#creating-an-array" title="Permanent link">&para;</a></h3>
 <div class="highlight"><pre><span></span><code><span class="c1">// This is how you declare an array of 50 elements each of type double</span>
-<span class="kt">double</span> <span class="n">DoubleArray</span><span class="p">[</span><span class="mi">50</span><span class="p">];</span>
+<span class="kt">double</span><span class="w"> </span><span class="n">DoubleArray</span><span class="p">[</span><span class="mi">50</span><span class="p">];</span>
 </code></pre></div>
 <h4 id="accessing-arrays-elements">Accessing Array's Elements<a class="headerlink" href="#accessing-arrays-elements" title="Permanent link">&para;</a></h4>
 <h5 id="first-element">First Element<a class="headerlink" href="#first-element" title="Permanent link">&para;</a></h5>
 <div class="highlight"><pre><span></span><code><span class="c1">// This access the first array </span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="n">DoubleArray</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">DoubleArray</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>0
 </code></pre></div>
 <h5 id="last-element">Last Element<a class="headerlink" href="#last-element" title="Permanent link">&para;</a></h5>
-<div class="highlight"><pre><span></span><code><span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="n">DoubleArray</span><span class="p">[</span><span class="mi">49</span><span class="p">]</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<div class="highlight"><pre><span></span><code><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">DoubleArray</span><span class="p">[</span><span class="mi">49</span><span class="p">]</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>0
 </code></pre></div>
 <h5 id="first-10-elements">First 10 Elements<a class="headerlink" href="#first-10-elements" title="Permanent link">&para;</a></h5>
 <div class="highlight"><pre><span></span><code><span class="c1">// In steps of 1</span>
-<span class="k">for</span> <span class="p">(</span><span class="kt">int</span> <span class="n">i</span><span class="o">=</span><span class="mi">0</span><span class="p">;</span> <span class="n">i</span><span class="o">&lt;</span><span class="mi">10</span><span class="p">;</span> <span class="n">i</span><span class="o">++</span><span class="p">)</span>
+<span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="kt">int</span><span class="w"> </span><span class="n">i</span><span class="o">=</span><span class="mi">0</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="o">&lt;</span><span class="mi">10</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="o">++</span><span class="p">)</span>
 <span class="p">{</span>
-    <span class="c1">// This is how you print a mix of characters and declared variables</span>
-    <span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Element &quot;</span> <span class="o">&lt;&lt;</span> <span class="n">i</span> <span class="o">&lt;&lt;</span> <span class="s">&quot; contains &quot;</span> <span class="o">&lt;&lt;</span>  <span class="n">DoubleArray</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<span class="w">    </span><span class="c1">// This is how you print a mix of characters and declared variables</span>
+<span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Element &quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot; contains &quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w">  </span><span class="n">DoubleArray</span><span class="p">[</span><span class="n">i</span><span class="p">]</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 <span class="p">}</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Element 0 contains 0
@@ -3123,10 +3263,10 @@ <h5 id="first-10-elements">First 10 Elements<a class="headerlink" href="#first-1
 Element 9 contains 0
 </code></pre></div>
 <div class="highlight"><pre><span></span><code><span class="c1">// In steps of 2</span>
-<span class="k">for</span> <span class="p">(</span><span class="kt">int</span> <span class="n">i</span><span class="o">=</span><span class="mi">0</span><span class="p">;</span> <span class="n">i</span><span class="o">&lt;</span><span class="mi">10</span><span class="p">;</span> <span class="n">i</span><span class="o">+=</span><span class="mi">2</span><span class="p">)</span>
+<span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="kt">int</span><span class="w"> </span><span class="n">i</span><span class="o">=</span><span class="mi">0</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="o">&lt;</span><span class="mi">10</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="o">+=</span><span class="mi">2</span><span class="p">)</span>
 <span class="p">{</span>
-    <span class="c1">// This is how you print a mix of characters and declared variables</span>
-    <span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Element &quot;</span> <span class="o">&lt;&lt;</span> <span class="n">i</span> <span class="o">&lt;&lt;</span> <span class="s">&quot; contains &quot;</span> <span class="o">&lt;&lt;</span>  <span class="n">DoubleArray</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<span class="w">    </span><span class="c1">// This is how you print a mix of characters and declared variables</span>
+<span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Element &quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot; contains &quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w">  </span><span class="n">DoubleArray</span><span class="p">[</span><span class="n">i</span><span class="p">]</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 <span class="p">}</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Element 0 contains 0
@@ -3137,28 +3277,28 @@ <h5 id="first-10-elements">First 10 Elements<a class="headerlink" href="#first-1
 </code></pre></div>
 <h5 id="going-beyond-the-arrays-length">Going Beyond The Array's Length<a class="headerlink" href="#going-beyond-the-arrays-length" title="Permanent link">&para;</a></h5>
 <p>This will return a warning that it's past the end of the array</p>
-<p><div class="highlight"><pre><span></span><code><span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="n">DoubleArray</span><span class="p">[</span><span class="mi">50</span><span class="p">]</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<p><div class="highlight"><pre><span></span><code><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">DoubleArray</span><span class="p">[</span><span class="mi">50</span><span class="p">]</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 </code></pre></div>
-<div class="highlight"><pre><span></span><code>input_line_36:2:15: warning: array index <span class="m">50</span> is past the end of the array <span class="o">(</span>which contains <span class="m">50</span> elements<span class="o">)</span>
-      <span class="o">[</span>-Warray-bounds<span class="o">]</span>
- std::cout <span class="s">&lt;&lt; DoubleArray[50] &lt;&lt; std::endl;</span>
+<div class="highlight"><pre><span></span><code>input_line_36:2:15:<span class="w"> </span>warning:<span class="w"> </span>array<span class="w"> </span>index<span class="w"> </span><span class="m">50</span><span class="w"> </span>is<span class="w"> </span>past<span class="w"> </span>the<span class="w"> </span>end<span class="w"> </span>of<span class="w"> </span>the<span class="w"> </span>array<span class="w"> </span><span class="o">(</span>which<span class="w"> </span>contains<span class="w"> </span><span class="m">50</span><span class="w"> </span>elements<span class="o">)</span>
+<span class="w">      </span><span class="o">[</span>-Warray-bounds<span class="o">]</span>
+<span class="w"> </span>std::cout<span class="w"> </span><span class="s">&lt;&lt; DoubleArray[50] &lt;&lt; std::endl;</span>
 <span class="s">              ^           ~~</span>
-<span class="s">input_line_32:3:1: note: array &#39;DoubleArray</span><span class="err">&#39;</span> declared here
-double DoubleArray<span class="o">[</span><span class="m">50</span><span class="o">]</span><span class="p">;</span>
+<span class="s">input_line_32:3:1: note: array &#39;DoubleArray</span><span class="err">&#39;</span><span class="w"> </span>declared<span class="w"> </span>here
+double<span class="w"> </span>DoubleArray<span class="o">[</span><span class="m">50</span><span class="o">]</span><span class="p">;</span>
 ^
 <span class="m">4</span>.94066e-323
 </code></pre></div></p>
 <h3 id="arrays-with-enumeration">Arrays with Enumeration<a class="headerlink" href="#arrays-with-enumeration" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code><span class="k">enum</span> <span class="nc">directionsNew</span> <span class="p">{</span><span class="n">up</span><span class="p">,</span> <span class="n">down</span><span class="p">,</span> <span class="n">left</span><span class="p">,</span> <span class="n">right</span><span class="p">,</span> <span class="n">individualDirections</span><span class="p">};</span>
+<div class="highlight"><pre><span></span><code><span class="k">enum</span><span class="w"> </span><span class="nc">directionsNew</span><span class="w"> </span><span class="p">{</span><span class="n">up</span><span class="p">,</span><span class="w"> </span><span class="n">down</span><span class="p">,</span><span class="w"> </span><span class="n">left</span><span class="p">,</span><span class="w"> </span><span class="n">right</span><span class="p">,</span><span class="w"> </span><span class="n">individualDirections</span><span class="p">};</span>
 
-<span class="kt">int</span> <span class="n">directionsArray</span><span class="p">[</span><span class="n">individualDirections</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">};</span>
+<span class="kt">int</span><span class="w"> </span><span class="n">directionsArray</span><span class="p">[</span><span class="n">individualDirections</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{</span><span class="mi">1</span><span class="p">,</span><span class="w"> </span><span class="mi">2</span><span class="p">,</span><span class="w"> </span><span class="mi">3</span><span class="p">,</span><span class="w"> </span><span class="mi">4</span><span class="p">};</span>
 
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Up value:</span><span class="se">\t</span><span class="s">&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">directionsArray</span><span class="p">[</span><span class="n">up</span><span class="p">];</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">Down value:</span><span class="se">\t</span><span class="s">&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">directionsArray</span><span class="p">[</span><span class="n">down</span><span class="p">];</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">Left value:</span><span class="se">\t</span><span class="s">&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">directionsArray</span><span class="p">[</span><span class="n">left</span><span class="p">];</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">Right value:</span><span class="se">\t</span><span class="s">&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">directionsArray</span><span class="p">[</span><span class="n">right</span><span class="p">];</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Up value:</span><span class="se">\t</span><span class="s">&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">directionsArray</span><span class="p">[</span><span class="n">up</span><span class="p">];</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;</span><span class="se">\n</span><span class="s">Down value:</span><span class="se">\t</span><span class="s">&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">directionsArray</span><span class="p">[</span><span class="n">down</span><span class="p">];</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;</span><span class="se">\n</span><span class="s">Left value:</span><span class="se">\t</span><span class="s">&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">directionsArray</span><span class="p">[</span><span class="n">left</span><span class="p">];</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;</span><span class="se">\n</span><span class="s">Right value:</span><span class="se">\t</span><span class="s">&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">directionsArray</span><span class="p">[</span><span class="n">right</span><span class="p">];</span>
 <span class="c1">// This is the number of elements in the array</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">Num value:</span><span class="se">\t</span><span class="s">&quot;</span> <span class="o">&lt;&lt;</span> <span class="k">sizeof</span><span class="p">(</span><span class="n">directionsArray</span><span class="p">)</span> <span class="o">/</span> <span class="k">sizeof</span><span class="p">(</span><span class="n">directionsArray</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;</span><span class="se">\n</span><span class="s">Num value:</span><span class="se">\t</span><span class="s">&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="k">sizeof</span><span class="p">(</span><span class="n">directionsArray</span><span class="p">)</span><span class="w"> </span><span class="o">/</span><span class="w"> </span><span class="k">sizeof</span><span class="p">(</span><span class="n">directionsArray</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Up value:   1
 Down value: 2
@@ -3169,22 +3309,22 @@ <h3 id="arrays-with-enumeration">Arrays with Enumeration<a class="headerlink" hr
 <h3 id="arrays-with-1-dimension-tensors">Arrays with &gt;1 Dimension (Tensors)<a class="headerlink" href="#arrays-with-1-dimension-tensors" title="Permanent link">&para;</a></h3>
 <h4 id="multi-dimension-array-with-numbers">Multi Dimension Array with Numbers<a class="headerlink" href="#multi-dimension-array-with-numbers" title="Permanent link">&para;</a></h4>
 <div class="highlight"><pre><span></span><code><span class="c1">// This is how you declare a multi-dimensional array of 5x5 elements each of type double</span>
-<span class="kt">double</span> <span class="n">multiDimArray</span><span class="p">[</span><span class="mi">5</span><span class="p">][</span><span class="mi">5</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
-    <span class="p">{</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">},</span>
-    <span class="p">{</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">},</span>
-    <span class="p">{</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">},</span>
-    <span class="p">{</span><span class="mi">4</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">},</span>
-    <span class="p">{</span><span class="mi">5</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">}</span>
+<span class="kt">double</span><span class="w"> </span><span class="n">multiDimArray</span><span class="p">[</span><span class="mi">5</span><span class="p">][</span><span class="mi">5</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{</span>
+<span class="w">    </span><span class="p">{</span><span class="mi">1</span><span class="p">,</span><span class="w"> </span><span class="mi">2</span><span class="p">,</span><span class="w"> </span><span class="mi">3</span><span class="p">,</span><span class="w"> </span><span class="mi">4</span><span class="p">,</span><span class="w"> </span><span class="mi">5</span><span class="p">},</span>
+<span class="w">    </span><span class="p">{</span><span class="mi">2</span><span class="p">,</span><span class="w"> </span><span class="mi">2</span><span class="p">,</span><span class="w"> </span><span class="mi">3</span><span class="p">,</span><span class="w"> </span><span class="mi">4</span><span class="p">,</span><span class="w"> </span><span class="mi">5</span><span class="p">},</span>
+<span class="w">    </span><span class="p">{</span><span class="mi">3</span><span class="p">,</span><span class="w"> </span><span class="mi">2</span><span class="p">,</span><span class="w"> </span><span class="mi">3</span><span class="p">,</span><span class="w"> </span><span class="mi">4</span><span class="p">,</span><span class="w"> </span><span class="mi">5</span><span class="p">},</span>
+<span class="w">    </span><span class="p">{</span><span class="mi">4</span><span class="p">,</span><span class="w"> </span><span class="mi">2</span><span class="p">,</span><span class="w"> </span><span class="mi">3</span><span class="p">,</span><span class="w"> </span><span class="mi">4</span><span class="p">,</span><span class="w"> </span><span class="mi">5</span><span class="p">},</span>
+<span class="w">    </span><span class="p">{</span><span class="mi">5</span><span class="p">,</span><span class="w"> </span><span class="mi">2</span><span class="p">,</span><span class="w"> </span><span class="mi">3</span><span class="p">,</span><span class="w"> </span><span class="mi">4</span><span class="p">,</span><span class="w"> </span><span class="mi">5</span><span class="p">}</span>
 <span class="p">};</span>
 
 <span class="c1">// Print each row of our 5x5 multi-dimensional array</span>
-<span class="k">for</span> <span class="p">(</span><span class="kt">int</span> <span class="n">i</span><span class="o">=</span><span class="mi">0</span><span class="p">;</span> <span class="n">i</span><span class="o">&lt;</span><span class="mi">5</span><span class="p">;</span> <span class="n">i</span><span class="o">++</span><span class="p">)</span>
+<span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="kt">int</span><span class="w"> </span><span class="n">i</span><span class="o">=</span><span class="mi">0</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="o">&lt;</span><span class="mi">5</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="o">++</span><span class="p">)</span>
 <span class="p">{</span>
-    <span class="k">for</span> <span class="p">(</span><span class="kt">int</span> <span class="n">j</span><span class="o">=</span><span class="mi">0</span><span class="p">;</span> <span class="n">j</span><span class="o">&lt;</span><span class="mi">5</span><span class="p">;</span> <span class="n">j</span><span class="o">++</span><span class="p">)</span>
-    <span class="p">{</span>
-        <span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="n">multiDimArray</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="n">j</span><span class="p">];</span>
-    <span class="p">};</span>
-    <span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<span class="w">    </span><span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="kt">int</span><span class="w"> </span><span class="n">j</span><span class="o">=</span><span class="mi">0</span><span class="p">;</span><span class="w"> </span><span class="n">j</span><span class="o">&lt;</span><span class="mi">5</span><span class="p">;</span><span class="w"> </span><span class="n">j</span><span class="o">++</span><span class="p">)</span>
+<span class="w">    </span><span class="p">{</span>
+<span class="w">        </span><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">multiDimArray</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="n">j</span><span class="p">];</span>
+<span class="w">    </span><span class="p">};</span>
+<span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 <span class="p">};</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>12345
@@ -3199,22 +3339,22 @@ <h4 id="multi-dimension-array-with-numbers">Multi Dimension Array with Numbers<a
 </code></pre></div>
 <h4 id="multi-dimension-array-with-characters">Multi Dimension Array with Characters<a class="headerlink" href="#multi-dimension-array-with-characters" title="Permanent link">&para;</a></h4>
 <div class="highlight"><pre><span></span><code><span class="c1">// This is how you declare a multi-dimensional array of 5x5 elements each of type char</span>
-<span class="kt">char</span> <span class="n">multiDimArrayChars</span><span class="p">[</span><span class="mi">5</span><span class="p">][</span><span class="mi">5</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
-    <span class="p">{</span><span class="sc">&#39;a&#39;</span><span class="p">,</span> <span class="sc">&#39;b&#39;</span><span class="p">,</span> <span class="sc">&#39;c&#39;</span><span class="p">,</span> <span class="sc">&#39;d&#39;</span><span class="p">,</span> <span class="sc">&#39;e&#39;</span><span class="p">},</span>
-    <span class="p">{</span><span class="sc">&#39;b&#39;</span><span class="p">,</span> <span class="sc">&#39;b&#39;</span><span class="p">,</span> <span class="sc">&#39;c&#39;</span><span class="p">,</span> <span class="sc">&#39;d&#39;</span><span class="p">,</span> <span class="sc">&#39;e&#39;</span><span class="p">},</span>
-    <span class="p">{</span><span class="sc">&#39;c&#39;</span><span class="p">,</span> <span class="sc">&#39;b&#39;</span><span class="p">,</span> <span class="sc">&#39;c&#39;</span><span class="p">,</span> <span class="sc">&#39;d&#39;</span><span class="p">,</span> <span class="sc">&#39;e&#39;</span><span class="p">},</span>
-    <span class="p">{</span><span class="sc">&#39;d&#39;</span><span class="p">,</span> <span class="sc">&#39;b&#39;</span><span class="p">,</span> <span class="sc">&#39;c&#39;</span><span class="p">,</span> <span class="sc">&#39;d&#39;</span><span class="p">,</span> <span class="sc">&#39;e&#39;</span><span class="p">},</span>
-    <span class="p">{</span><span class="sc">&#39;e&#39;</span><span class="p">,</span> <span class="sc">&#39;b&#39;</span><span class="p">,</span> <span class="sc">&#39;c&#39;</span><span class="p">,</span> <span class="sc">&#39;d&#39;</span><span class="p">,</span> <span class="sc">&#39;e&#39;</span><span class="p">},</span>
+<span class="kt">char</span><span class="w"> </span><span class="n">multiDimArrayChars</span><span class="p">[</span><span class="mi">5</span><span class="p">][</span><span class="mi">5</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{</span>
+<span class="w">    </span><span class="p">{</span><span class="sc">&#39;a&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;b&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;c&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;d&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;e&#39;</span><span class="p">},</span>
+<span class="w">    </span><span class="p">{</span><span class="sc">&#39;b&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;b&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;c&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;d&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;e&#39;</span><span class="p">},</span>
+<span class="w">    </span><span class="p">{</span><span class="sc">&#39;c&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;b&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;c&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;d&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;e&#39;</span><span class="p">},</span>
+<span class="w">    </span><span class="p">{</span><span class="sc">&#39;d&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;b&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;c&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;d&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;e&#39;</span><span class="p">},</span>
+<span class="w">    </span><span class="p">{</span><span class="sc">&#39;e&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;b&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;c&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;d&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;e&#39;</span><span class="p">},</span>
 <span class="p">};</span>
 
 <span class="c1">// Print each row of our 5x5 multi-dimensional array</span>
-<span class="k">for</span> <span class="p">(</span><span class="kt">int</span> <span class="n">i</span><span class="o">=</span><span class="mi">0</span><span class="p">;</span> <span class="n">i</span><span class="o">&lt;</span><span class="mi">5</span><span class="p">;</span> <span class="n">i</span><span class="o">++</span><span class="p">)</span>
+<span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="kt">int</span><span class="w"> </span><span class="n">i</span><span class="o">=</span><span class="mi">0</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="o">&lt;</span><span class="mi">5</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="o">++</span><span class="p">)</span>
 <span class="p">{</span>
-    <span class="k">for</span> <span class="p">(</span><span class="kt">int</span> <span class="n">j</span><span class="o">=</span><span class="mi">0</span><span class="p">;</span> <span class="n">j</span><span class="o">&lt;</span><span class="mi">5</span><span class="p">;</span> <span class="n">j</span><span class="o">++</span><span class="p">)</span>
-    <span class="p">{</span>
-        <span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="n">multiDimArrayChars</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="n">j</span><span class="p">];</span>
-    <span class="p">};</span>
-    <span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<span class="w">    </span><span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="kt">int</span><span class="w"> </span><span class="n">j</span><span class="o">=</span><span class="mi">0</span><span class="p">;</span><span class="w"> </span><span class="n">j</span><span class="o">&lt;</span><span class="mi">5</span><span class="p">;</span><span class="w"> </span><span class="n">j</span><span class="o">++</span><span class="p">)</span>
+<span class="w">    </span><span class="p">{</span>
+<span class="w">        </span><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">multiDimArrayChars</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="n">j</span><span class="p">];</span>
+<span class="w">    </span><span class="p">};</span>
+<span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 <span class="p">};</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>abcde
@@ -3229,17 +3369,17 @@ <h4 id="multi-dimension-array-with-characters">Multi Dimension Array with Charac
 </code></pre></div>
 <h3 id="copy-arrays">Copy Arrays<a class="headerlink" href="#copy-arrays" title="Permanent link">&para;</a></h3>
 <h4 id="copy-number-arrays">Copy Number Arrays<a class="headerlink" href="#copy-number-arrays" title="Permanent link">&para;</a></h4>
-<div class="highlight"><pre><span></span><code><span class="kt">double</span> <span class="n">ArrayNumOne</span><span class="p">[]</span> <span class="o">=</span> <span class="p">{</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">};</span>
-<span class="kt">double</span> <span class="n">ArrayNumTwo</span><span class="p">[</span><span class="mi">5</span><span class="p">];</span>
+<div class="highlight"><pre><span></span><code><span class="kt">double</span><span class="w"> </span><span class="n">ArrayNumOne</span><span class="p">[]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{</span><span class="mi">1</span><span class="p">,</span><span class="w"> </span><span class="mi">2</span><span class="p">,</span><span class="w"> </span><span class="mi">3</span><span class="p">,</span><span class="w"> </span><span class="mi">4</span><span class="p">,</span><span class="w"> </span><span class="mi">5</span><span class="p">};</span>
+<span class="kt">double</span><span class="w"> </span><span class="n">ArrayNumTwo</span><span class="p">[</span><span class="mi">5</span><span class="p">];</span>
 
 <span class="c1">// Use namespace std so it&#39;s cleaner</span>
-<span class="k">using</span> <span class="k">namespace</span> <span class="nn">std</span><span class="p">;</span>
+<span class="k">using</span><span class="w"> </span><span class="k">namespace</span><span class="w"> </span><span class="nn">std</span><span class="p">;</span>
 
 <span class="c1">// Copy array with copy()</span>
-<span class="n">copy</span><span class="p">(</span><span class="n">begin</span><span class="p">(</span><span class="n">ArrayNumOne</span><span class="p">),</span> <span class="n">end</span><span class="p">(</span><span class="n">ArrayNumOne</span><span class="p">),</span> <span class="n">begin</span><span class="p">(</span><span class="n">ArrayNumTwo</span><span class="p">));</span>
+<span class="n">copy</span><span class="p">(</span><span class="n">begin</span><span class="p">(</span><span class="n">ArrayNumOne</span><span class="p">),</span><span class="w"> </span><span class="n">end</span><span class="p">(</span><span class="n">ArrayNumOne</span><span class="p">),</span><span class="w"> </span><span class="n">begin</span><span class="p">(</span><span class="n">ArrayNumTwo</span><span class="p">));</span>
 
 <span class="c1">// Print double type array with copy()</span>
-<span class="n">copy</span><span class="p">(</span><span class="n">begin</span><span class="p">(</span><span class="n">ArrayNumTwo</span><span class="p">),</span> <span class="n">end</span><span class="p">(</span><span class="n">ArrayNumTwo</span><span class="p">),</span> <span class="n">ostream_iterator</span><span class="o">&lt;</span><span class="kt">double</span><span class="o">&gt;</span><span class="p">(</span><span class="n">cout</span><span class="p">,</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">));</span>
+<span class="n">copy</span><span class="p">(</span><span class="n">begin</span><span class="p">(</span><span class="n">ArrayNumTwo</span><span class="p">),</span><span class="w"> </span><span class="n">end</span><span class="p">(</span><span class="n">ArrayNumTwo</span><span class="p">),</span><span class="w"> </span><span class="n">ostream_iterator</span><span class="o">&lt;</span><span class="kt">double</span><span class="o">&gt;</span><span class="p">(</span><span class="n">cout</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">));</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>1
 2
@@ -3248,17 +3388,17 @@ <h4 id="copy-number-arrays">Copy Number Arrays<a class="headerlink" href="#copy-
 5
 </code></pre></div>
 <h4 id="copy-string-arrays">Copy String Arrays<a class="headerlink" href="#copy-string-arrays" title="Permanent link">&para;</a></h4>
-<div class="highlight"><pre><span></span><code><span class="kt">char</span> <span class="n">ArrayCharOne</span><span class="p">[]</span> <span class="o">=</span> <span class="p">{</span><span class="sc">&#39;a&#39;</span><span class="p">,</span> <span class="sc">&#39;b&#39;</span><span class="p">,</span> <span class="sc">&#39;c&#39;</span><span class="p">,</span> <span class="sc">&#39;d&#39;</span><span class="p">,</span> <span class="sc">&#39;e&#39;</span><span class="p">};</span>
-<span class="kt">char</span> <span class="n">ArrayCharTwo</span><span class="p">[</span><span class="mi">5</span><span class="p">];</span>
+<div class="highlight"><pre><span></span><code><span class="kt">char</span><span class="w"> </span><span class="n">ArrayCharOne</span><span class="p">[]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{</span><span class="sc">&#39;a&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;b&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;c&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;d&#39;</span><span class="p">,</span><span class="w"> </span><span class="sc">&#39;e&#39;</span><span class="p">};</span>
+<span class="kt">char</span><span class="w"> </span><span class="n">ArrayCharTwo</span><span class="p">[</span><span class="mi">5</span><span class="p">];</span>
 
 <span class="c1">// Use namespace std so it&#39;s cleaner</span>
-<span class="k">using</span> <span class="k">namespace</span> <span class="nn">std</span><span class="p">;</span>
+<span class="k">using</span><span class="w"> </span><span class="k">namespace</span><span class="w"> </span><span class="nn">std</span><span class="p">;</span>
 
 <span class="c1">// Copy array with copy()</span>
-<span class="n">copy</span><span class="p">(</span><span class="n">begin</span><span class="p">(</span><span class="n">ArrayCharOne</span><span class="p">),</span> <span class="n">end</span><span class="p">(</span><span class="n">ArrayCharOne</span><span class="p">),</span> <span class="n">begin</span><span class="p">(</span><span class="n">ArrayCharTwo</span><span class="p">));</span>
+<span class="n">copy</span><span class="p">(</span><span class="n">begin</span><span class="p">(</span><span class="n">ArrayCharOne</span><span class="p">),</span><span class="w"> </span><span class="n">end</span><span class="p">(</span><span class="n">ArrayCharOne</span><span class="p">),</span><span class="w"> </span><span class="n">begin</span><span class="p">(</span><span class="n">ArrayCharTwo</span><span class="p">));</span>
 
 <span class="c1">// Print char type array with copy()</span>
-<span class="n">copy</span><span class="p">(</span><span class="n">begin</span><span class="p">(</span><span class="n">ArrayCharTwo</span><span class="p">),</span> <span class="n">end</span><span class="p">(</span><span class="n">ArrayCharTwo</span><span class="p">),</span> <span class="n">ostream_iterator</span><span class="o">&lt;</span><span class="kt">char</span><span class="o">&gt;</span><span class="p">(</span><span class="n">cout</span><span class="p">,</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">));</span>
+<span class="n">copy</span><span class="p">(</span><span class="n">begin</span><span class="p">(</span><span class="n">ArrayCharTwo</span><span class="p">),</span><span class="w"> </span><span class="n">end</span><span class="p">(</span><span class="n">ArrayCharTwo</span><span class="p">),</span><span class="w"> </span><span class="n">ostream_iterator</span><span class="o">&lt;</span><span class="kt">char</span><span class="o">&gt;</span><span class="p">(</span><span class="n">cout</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">));</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>a
 b
@@ -3271,26 +3411,26 @@ <h2 id="mathematical-operators">Mathematical Operators<a class="headerlink" href
 <li>Add, subtract, multiply, divide and modulus</li>
 </ul>
 <h3 id="add">Add<a class="headerlink" href="#add" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code><span class="kt">double</span> <span class="n">addStatement</span> <span class="o">=</span> <span class="mi">5</span> <span class="o">+</span> <span class="mi">5</span><span class="p">;</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="n">addStatement</span><span class="p">;</span>
+<div class="highlight"><pre><span></span><code><span class="kt">double</span><span class="w"> </span><span class="n">addStatement</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">5</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">5</span><span class="p">;</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">addStatement</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>10
 </code></pre></div>
 <h3 id="subtract">Subtract<a class="headerlink" href="#subtract" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code><span class="kt">double</span> <span class="n">subtractStatement</span> <span class="o">=</span> <span class="mi">5</span> <span class="o">-</span> <span class="mi">5</span><span class="p">;</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="n">subtractStatement</span><span class="p">;</span>
+<div class="highlight"><pre><span></span><code><span class="kt">double</span><span class="w"> </span><span class="n">subtractStatement</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">5</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="mi">5</span><span class="p">;</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">subtractStatement</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>0
 </code></pre></div>
 <h3 id="divide">Divide<a class="headerlink" href="#divide" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code><span class="kt">double</span> <span class="n">divideStatement</span> <span class="o">=</span> <span class="mi">5</span> <span class="o">/</span> <span class="mi">5</span><span class="p">;</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="n">divideStatement</span><span class="p">;</span>
+<div class="highlight"><pre><span></span><code><span class="kt">double</span><span class="w"> </span><span class="n">divideStatement</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">5</span><span class="w"> </span><span class="o">/</span><span class="w"> </span><span class="mi">5</span><span class="p">;</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">divideStatement</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>1
 </code></pre></div>
 <h3 id="multiply">Multiply<a class="headerlink" href="#multiply" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code><span class="kt">double</span> <span class="n">multiplyStatement</span> <span class="o">=</span> <span class="mi">5</span> <span class="o">*</span> <span class="mi">5</span><span class="p">;</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="n">multiplyStatement</span><span class="p">;</span>
+<div class="highlight"><pre><span></span><code><span class="kt">double</span><span class="w"> </span><span class="n">multiplyStatement</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">5</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="mi">5</span><span class="p">;</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">multiplyStatement</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>25
 </code></pre></div>
@@ -3298,8 +3438,8 @@ <h3 id="modulus">Modulus<a class="headerlink" href="#modulus" title="Permanent l
 <ul>
 <li>Gets the remainder of the division</li>
 </ul>
-<div class="highlight"><pre><span></span><code><span class="kt">double</span> <span class="n">modulusStatement</span> <span class="o">=</span> <span class="mi">8</span> <span class="o">%</span> <span class="mi">5</span><span class="p">;</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="n">modulusStatement</span><span class="p">;</span>
+<div class="highlight"><pre><span></span><code><span class="kt">double</span><span class="w"> </span><span class="n">modulusStatement</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">8</span><span class="w"> </span><span class="o">%</span><span class="w"> </span><span class="mi">5</span><span class="p">;</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">modulusStatement</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>3
 </code></pre></div>
@@ -3307,17 +3447,17 @@ <h3 id="exponent">Exponent<a class="headerlink" href="#exponent" title="Permanen
 <ul>
 <li>Base to the power of something, this requires a new package called <code>&lt;cmath&gt;</code> that we want to include.</li>
 </ul>
-<div class="highlight"><pre><span></span><code><span class="cp">#include</span> <span class="cpf">&lt;cmath&gt;</span><span class="cp"></span>
+<div class="highlight"><pre><span></span><code><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;cmath&gt;</span>
 
-<span class="kt">void</span> <span class="nf">SquareNumber</span><span class="p">(</span><span class="kt">int</span> <span class="n">baseNum</span><span class="p">,</span> <span class="kt">int</span> <span class="n">exponentNum</span><span class="p">)</span>
+<span class="kt">void</span><span class="w"> </span><span class="nf">SquareNumber</span><span class="p">(</span><span class="kt">int</span><span class="w"> </span><span class="n">baseNum</span><span class="p">,</span><span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">exponentNum</span><span class="p">)</span>
 <span class="p">{</span>
-    <span class="c1">// Square the locally scoped variable with 2</span>
-    <span class="kt">int</span> <span class="n">squaredNumber</span><span class="p">;</span>
-    <span class="n">squaredNumber</span> <span class="o">=</span> <span class="n">pow</span><span class="p">(</span><span class="n">baseNum</span><span class="p">,</span> <span class="n">exponentNum</span><span class="p">);</span>
-    <span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Base of 2 with exponent of 2 gives: &quot;</span> <span class="o">&lt;&lt;</span> <span class="n">squaredNumber</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<span class="w">    </span><span class="c1">// Square the locally scoped variable with 2</span>
+<span class="w">    </span><span class="kt">int</span><span class="w"> </span><span class="n">squaredNumber</span><span class="p">;</span>
+<span class="w">    </span><span class="n">squaredNumber</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">pow</span><span class="p">(</span><span class="n">baseNum</span><span class="p">,</span><span class="w"> </span><span class="n">exponentNum</span><span class="p">);</span>
+<span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Base of 2 with exponent of 2 gives: &quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">squaredNumber</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 <span class="p">}</span>
 
-<span class="n">SquareNumber</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
+<span class="n">SquareNumber</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span><span class="w"> </span><span class="mi">2</span><span class="p">)</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Base of 2 with exponent of 2 gives: 4
 </code></pre></div>
@@ -3327,21 +3467,21 @@ <h3 id="incrementingdecrementing">Incrementing/Decrementing<a class="headerlink"
 </ul>
 <h4 id="methods">Methods<a class="headerlink" href="#methods" title="Permanent link">&para;</a></h4>
 <h5 id="method-1">Method 1<a class="headerlink" href="#method-1" title="Permanent link">&para;</a></h5>
-<div class="highlight"><pre><span></span><code><span class="kt">double</span> <span class="n">idx</span> <span class="o">=</span> <span class="mi">1</span><span class="p">;</span>
+<div class="highlight"><pre><span></span><code><span class="kt">double</span><span class="w"> </span><span class="n">idx</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">1</span><span class="p">;</span>
 <span class="n">idx</span><span class="o">++</span><span class="p">;</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="n">idx</span><span class="p">;</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">idx</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>2
 </code></pre></div>
 <h5 id="method-2">Method 2<a class="headerlink" href="#method-2" title="Permanent link">&para;</a></h5>
-<div class="highlight"><pre><span></span><code><span class="n">idx</span> <span class="o">+=</span> <span class="mi">1</span><span class="p">;</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="n">idx</span><span class="p">;</span>
+<div class="highlight"><pre><span></span><code><span class="n">idx</span><span class="w"> </span><span class="o">+=</span><span class="w"> </span><span class="mi">1</span><span class="p">;</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">idx</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>3
 </code></pre></div>
 <h5 id="method-3">Method 3<a class="headerlink" href="#method-3" title="Permanent link">&para;</a></h5>
-<div class="highlight"><pre><span></span><code><span class="n">idx</span> <span class="o">=</span> <span class="n">idx</span> <span class="o">+</span> <span class="mi">1</span><span class="p">;</span>
-<span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="n">idx</span><span class="p">;</span>
+<div class="highlight"><pre><span></span><code><span class="n">idx</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">idx</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="p">;</span>
+<span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">idx</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>4
 </code></pre></div>
@@ -3352,19 +3492,19 @@ <h5 id="prefix">Prefix<a class="headerlink" href="#prefix" title="Permanent link
 <li>Summary: both variables will have the same values</li>
 </ul>
 <div class="highlight"><pre><span></span><code><span class="c1">// Instantiate</span>
-<span class="kt">double</span> <span class="n">a</span> <span class="o">=</span> <span class="mi">1</span><span class="p">;</span>
-<span class="kt">double</span> <span class="n">b</span> <span class="o">=</span> <span class="mi">1</span><span class="p">;</span>
+<span class="kt">double</span><span class="w"> </span><span class="n">a</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">1</span><span class="p">;</span>
+<span class="kt">double</span><span class="w"> </span><span class="n">b</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">1</span><span class="p">;</span>
 
 <span class="c1">// Print original values</span>
-<span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Old a:</span><span class="se">\t</span><span class="s">&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">a</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
-<span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Old b:</span><span class="se">\t</span><span class="s">&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">b</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
+<span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Old a:</span><span class="se">\t</span><span class="s">&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">a</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
+<span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Old b:</span><span class="se">\t</span><span class="s">&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">b</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
 
 <span class="c1">// Prefix increment</span>
-<span class="n">a</span> <span class="o">=</span> <span class="o">++</span><span class="n">b</span><span class="p">;</span>
+<span class="n">a</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="o">++</span><span class="n">b</span><span class="p">;</span>
 
 <span class="c1">// Print new values</span>
-<span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;New a:</span><span class="se">\t</span><span class="s">&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">a</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
-<span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;New b:</span><span class="se">\t</span><span class="s">&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">b</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
+<span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;New a:</span><span class="se">\t</span><span class="s">&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">a</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
+<span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;New b:</span><span class="se">\t</span><span class="s">&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">b</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Old a:  1
 Old b:  1
@@ -3377,19 +3517,19 @@ <h5 id="postfix">Postfix<a class="headerlink" href="#postfix" title="Permanent l
 <li>Summary: incremented variable will change but not the variable it was assigned to</li>
 </ul>
 <div class="highlight"><pre><span></span><code><span class="c1">// Instantiate</span>
-<span class="kt">double</span> <span class="n">c</span> <span class="o">=</span> <span class="mi">2</span><span class="p">;</span>
-<span class="kt">double</span> <span class="n">d</span> <span class="o">=</span> <span class="mi">2</span><span class="p">;</span>
+<span class="kt">double</span><span class="w"> </span><span class="n">c</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">2</span><span class="p">;</span>
+<span class="kt">double</span><span class="w"> </span><span class="n">d</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">2</span><span class="p">;</span>
 
 <span class="c1">// Print original values</span>
-<span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Old c:</span><span class="se">\t</span><span class="s">&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">c</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
-<span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Old d:</span><span class="se">\t</span><span class="s">&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">d</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
+<span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Old c:</span><span class="se">\t</span><span class="s">&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">c</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
+<span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Old d:</span><span class="se">\t</span><span class="s">&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">d</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
 
 <span class="c1">// Prefix increment</span>
-<span class="n">c</span> <span class="o">=</span> <span class="n">d</span><span class="o">--</span><span class="p">;</span>
+<span class="n">c</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">d</span><span class="o">--</span><span class="p">;</span>
 
 <span class="c1">// Print new values, notice how only d decremented? c which is what d is assigned to doesn&#39;t change.</span>
-<span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;New c:</span><span class="se">\t</span><span class="s">&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">c</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
-<span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;New d:</span><span class="se">\t</span><span class="s">&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">d</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
+<span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;New c:</span><span class="se">\t</span><span class="s">&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">c</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
+<span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;New d:</span><span class="se">\t</span><span class="s">&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">d</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Old c:  2
 Old d:  2
@@ -3398,39 +3538,39 @@ <h5 id="postfix">Postfix<a class="headerlink" href="#postfix" title="Permanent l
 </code></pre></div>
 <h2 id="conditional-statements">Conditional Statements<a class="headerlink" href="#conditional-statements" title="Permanent link">&para;</a></h2>
 <h4 id="if">If<a class="headerlink" href="#if" title="Permanent link">&para;</a></h4>
-<div class="highlight"><pre><span></span><code><span class="kt">int</span> <span class="n">maxValue</span> <span class="o">=</span> <span class="mi">10</span><span class="p">;</span>
+<div class="highlight"><pre><span></span><code><span class="kt">int</span><span class="w"> </span><span class="n">maxValue</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">10</span><span class="p">;</span>
 
 <span class="c1">// Increment till 10</span>
-<span class="k">for</span> <span class="p">(</span><span class="kt">int</span> <span class="n">i</span><span class="o">=</span><span class="mi">0</span><span class="p">;</span> <span class="n">i</span><span class="o">&lt;=</span><span class="mi">10</span><span class="p">;</span> <span class="n">i</span><span class="o">+=</span><span class="mi">2</span><span class="p">)</span>
+<span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="kt">int</span><span class="w"> </span><span class="n">i</span><span class="o">=</span><span class="mi">0</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="o">&lt;=</span><span class="mi">10</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="o">+=</span><span class="mi">2</span><span class="p">)</span>
 <span class="p">{</span>
-    <span class="c1">// Stop if the number reaches 10 (inclusive)</span>
-    <span class="k">if</span> <span class="p">(</span><span class="n">i</span> <span class="o">==</span> <span class="n">maxValue</span><span class="p">)</span>
-    <span class="p">{</span>
-        <span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Reached max value!&quot;</span><span class="p">;</span>
-        <span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">Value is &quot;</span> <span class="o">&lt;&lt;</span> <span class="n">i</span> <span class="o">&lt;&lt;</span> <span class="n">endl</span><span class="p">;</span>
-    <span class="p">};</span>
+<span class="w">    </span><span class="c1">// Stop if the number reaches 10 (inclusive)</span>
+<span class="w">    </span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="n">i</span><span class="w"> </span><span class="o">==</span><span class="w"> </span><span class="n">maxValue</span><span class="p">)</span>
+<span class="w">    </span><span class="p">{</span>
+<span class="w">        </span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Reached max value!&quot;</span><span class="p">;</span>
+<span class="w">        </span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;</span><span class="se">\n</span><span class="s">Value is &quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">endl</span><span class="p">;</span>
+<span class="w">    </span><span class="p">};</span>
 <span class="p">};</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Reached max value!
 Value is 10
 </code></pre></div>
 <h4 id="else">Else<a class="headerlink" href="#else" title="Permanent link">&para;</a></h4>
-<div class="highlight"><pre><span></span><code><span class="kt">int</span> <span class="n">newMaxValue</span> <span class="o">=</span> <span class="mi">20</span><span class="p">;</span>
+<div class="highlight"><pre><span></span><code><span class="kt">int</span><span class="w"> </span><span class="n">newMaxValue</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">20</span><span class="p">;</span>
 
 <span class="c1">// Increment till 10</span>
-<span class="k">for</span> <span class="p">(</span><span class="kt">int</span> <span class="n">i</span><span class="o">=</span><span class="mi">0</span><span class="p">;</span> <span class="n">i</span><span class="o">&lt;=</span><span class="mi">10</span><span class="p">;</span> <span class="n">i</span><span class="o">+=</span><span class="mi">2</span><span class="p">)</span>
+<span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="kt">int</span><span class="w"> </span><span class="n">i</span><span class="o">=</span><span class="mi">0</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="o">&lt;=</span><span class="mi">10</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="o">+=</span><span class="mi">2</span><span class="p">)</span>
 <span class="p">{</span>
-    <span class="c1">// Stop if the number reaches 10 (inclusive)</span>
-    <span class="k">if</span> <span class="p">(</span><span class="n">i</span> <span class="o">==</span> <span class="n">newMaxValue</span><span class="p">)</span>
-    <span class="p">{</span>
-        <span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Reached max value!&quot;</span><span class="p">;</span>
-        <span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">Value is &quot;</span> <span class="o">&lt;&lt;</span> <span class="n">i</span> <span class="o">&lt;&lt;</span> <span class="n">endl</span><span class="p">;</span>
-    <span class="p">}</span>
-    <span class="c1">// Else print current value</span>
-    <span class="k">else</span>
-    <span class="p">{</span>
-        <span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">Current Value is &quot;</span> <span class="o">&lt;&lt;</span> <span class="n">i</span> <span class="o">&lt;&lt;</span> <span class="n">endl</span><span class="p">;</span>
-    <span class="p">}</span>
+<span class="w">    </span><span class="c1">// Stop if the number reaches 10 (inclusive)</span>
+<span class="w">    </span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="n">i</span><span class="w"> </span><span class="o">==</span><span class="w"> </span><span class="n">newMaxValue</span><span class="p">)</span>
+<span class="w">    </span><span class="p">{</span>
+<span class="w">        </span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Reached max value!&quot;</span><span class="p">;</span>
+<span class="w">        </span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;</span><span class="se">\n</span><span class="s">Value is &quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">endl</span><span class="p">;</span>
+<span class="w">    </span><span class="p">}</span>
+<span class="w">    </span><span class="c1">// Else print current value</span>
+<span class="w">    </span><span class="k">else</span>
+<span class="w">    </span><span class="p">{</span>
+<span class="w">        </span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;</span><span class="se">\n</span><span class="s">Current Value is &quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">endl</span><span class="p">;</span>
+<span class="w">    </span><span class="p">}</span>
 <span class="p">}</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Current Value is 0
@@ -3447,21 +3587,21 @@ <h4 id="else">Else<a class="headerlink" href="#else" title="Permanent link">&par
 </code></pre></div>
 <h2 id="logical-operators">Logical Operators<a class="headerlink" href="#logical-operators" title="Permanent link">&para;</a></h2>
 <h4 id="and">And<a class="headerlink" href="#and" title="Permanent link">&para;</a></h4>
-<div class="highlight"><pre><span></span><code><span class="kt">int</span> <span class="n">varOneNew</span> <span class="o">=</span> <span class="mi">10</span><span class="p">;</span>
-<span class="kt">int</span> <span class="n">varTwo</span> <span class="o">=</span> <span class="mi">10</span><span class="p">;</span>
-<span class="kt">int</span> <span class="n">varCheckOne</span> <span class="o">=</span> <span class="mi">10</span><span class="p">;</span>
-<span class="kt">int</span> <span class="n">varCheckTwo</span> <span class="o">=</span> <span class="mi">5</span><span class="p">;</span>
+<div class="highlight"><pre><span></span><code><span class="kt">int</span><span class="w"> </span><span class="n">varOneNew</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">10</span><span class="p">;</span>
+<span class="kt">int</span><span class="w"> </span><span class="n">varTwo</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">10</span><span class="p">;</span>
+<span class="kt">int</span><span class="w"> </span><span class="n">varCheckOne</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">10</span><span class="p">;</span>
+<span class="kt">int</span><span class="w"> </span><span class="n">varCheckTwo</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">5</span><span class="p">;</span>
 
 <span class="c1">// This should print out</span>
-<span class="k">if</span> <span class="p">((</span><span class="n">varOneNew</span> <span class="o">==</span> <span class="n">varCheckOne</span><span class="p">)</span> <span class="o">&amp;&amp;</span> <span class="p">(</span><span class="n">varTwo</span> <span class="o">==</span> <span class="n">varCheckOne</span><span class="p">))</span>
+<span class="k">if</span><span class="w"> </span><span class="p">((</span><span class="n">varOneNew</span><span class="w"> </span><span class="o">==</span><span class="w"> </span><span class="n">varCheckOne</span><span class="p">)</span><span class="w"> </span><span class="o">&amp;&amp;</span><span class="w"> </span><span class="p">(</span><span class="n">varTwo</span><span class="w"> </span><span class="o">==</span><span class="w"> </span><span class="n">varCheckOne</span><span class="p">))</span>
 <span class="p">{</span>
-    <span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Both values equal to 10!&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Both values equal to 10!&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 <span class="p">}</span>
 
 <span class="c1">// This should not print out as varTwo does not equal to 5</span>
-<span class="k">if</span> <span class="p">((</span><span class="n">varOneNew</span> <span class="o">==</span> <span class="n">varCheckOne</span><span class="p">)</span> <span class="o">&amp;&amp;</span> <span class="p">(</span><span class="n">varTwo</span> <span class="o">==</span> <span class="n">varCheckTwo</span><span class="p">))</span>
+<span class="k">if</span><span class="w"> </span><span class="p">((</span><span class="n">varOneNew</span><span class="w"> </span><span class="o">==</span><span class="w"> </span><span class="n">varCheckOne</span><span class="p">)</span><span class="w"> </span><span class="o">&amp;&amp;</span><span class="w"> </span><span class="p">(</span><span class="n">varTwo</span><span class="w"> </span><span class="o">==</span><span class="w"> </span><span class="n">varCheckTwo</span><span class="p">))</span>
 <span class="p">{</span>
-    <span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;VarOneNew equals to 10, VarTwo equals to 5&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;VarOneNew equals to 10, VarTwo equals to 5&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 <span class="p">}</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Both values equal to 10!
@@ -3469,31 +3609,31 @@ <h4 id="and">And<a class="headerlink" href="#and" title="Permanent link">&para;<
 <h4 id="or">Or<a class="headerlink" href="#or" title="Permanent link">&para;</a></h4>
 <div class="highlight"><pre><span></span><code><span class="c1">// On the contrary, this exact same statement would print out</span>
 <span class="c1">// as VarOne is equal to 10 and we are using an OR operator</span>
-<span class="k">if</span> <span class="p">((</span><span class="n">varOneNew</span> <span class="o">==</span> <span class="n">varCheckOne</span><span class="p">)</span> <span class="o">||</span> <span class="p">(</span><span class="n">varTwo</span> <span class="o">==</span> <span class="n">varCheckTwo</span><span class="p">))</span>
+<span class="k">if</span><span class="w"> </span><span class="p">((</span><span class="n">varOneNew</span><span class="w"> </span><span class="o">==</span><span class="w"> </span><span class="n">varCheckOne</span><span class="p">)</span><span class="w"> </span><span class="o">||</span><span class="w"> </span><span class="p">(</span><span class="n">varTwo</span><span class="w"> </span><span class="o">==</span><span class="w"> </span><span class="n">varCheckTwo</span><span class="p">))</span>
 <span class="p">{</span>
-    <span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;VarOneNew equals to 10 or VarTwo equals to 5&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;VarOneNew equals to 10 or VarTwo equals to 5&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 <span class="p">}</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>VarOneNew equals to 10 or VarTwo equals to 5
 </code></pre></div>
 <h4 id="not">Not<a class="headerlink" href="#not" title="Permanent link">&para;</a></h4>
 <div class="highlight"><pre><span></span><code><span class="c1">// This would print out as VarTwo is not equal to 5</span>
-<span class="k">if</span> <span class="p">(</span><span class="n">varTwo</span> <span class="o">!=</span> <span class="n">varCheckTwo</span><span class="p">)</span>
+<span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="n">varTwo</span><span class="w"> </span><span class="o">!=</span><span class="w"> </span><span class="n">varCheckTwo</span><span class="p">)</span>
 <span class="p">{</span>
-    <span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;VarTwo (10) is not equal to VarCheckTwo (5).&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
+<span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;VarTwo (10) is not equal to VarCheckTwo (5).&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span>
 <span class="p">}</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>VarTwo (10) is not equal to VarCheckTwo (5).
 </code></pre></div>
 <h2 id="getting-user-input">Getting User Input<a class="headerlink" href="#getting-user-input" title="Permanent link">&para;</a></h2>
-<div class="highlight"><pre><span></span><code><span class="k">using</span> <span class="k">namespace</span> <span class="nn">std</span><span class="p">;</span>
-<span class="kt">long</span> <span class="kt">double</span> <span class="n">inputOne</span><span class="p">,</span> <span class="n">inputTwo</span><span class="p">;</span>
-<span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;This program multiplies 2 given numbers</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
-<span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Enter first number: </span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
-<span class="n">cin</span> <span class="o">&gt;&gt;</span> <span class="n">inputOne</span><span class="p">;</span>
-<span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Enter second number: </span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
-<span class="n">cin</span> <span class="o">&gt;&gt;</span> <span class="n">inputTwo</span><span class="p">;</span>
-<span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Multiplication value: &quot;</span> <span class="o">&lt;&lt;</span> <span class="n">inputOne</span> <span class="o">*</span> <span class="n">inputTwo</span> <span class="o">&lt;&lt;</span> <span class="n">endl</span><span class="p">;</span>
+<div class="highlight"><pre><span></span><code><span class="k">using</span><span class="w"> </span><span class="k">namespace</span><span class="w"> </span><span class="nn">std</span><span class="p">;</span>
+<span class="kt">long</span><span class="w"> </span><span class="kt">double</span><span class="w"> </span><span class="n">inputOne</span><span class="p">,</span><span class="w"> </span><span class="n">inputTwo</span><span class="p">;</span>
+<span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;This program multiplies 2 given numbers</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
+<span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Enter first number: </span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
+<span class="n">cin</span><span class="w"> </span><span class="o">&gt;&gt;</span><span class="w"> </span><span class="n">inputOne</span><span class="p">;</span>
+<span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Enter second number: </span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>
+<span class="n">cin</span><span class="w"> </span><span class="o">&gt;&gt;</span><span class="w"> </span><span class="n">inputTwo</span><span class="p">;</span>
+<span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Multiplication value: &quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">inputOne</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="n">inputTwo</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">endl</span><span class="p">;</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>This program multiplies 2 given numbers
 Enter first number: 
@@ -3504,9 +3644,9 @@ <h2 id="getting-user-input">Getting User Input<a class="headerlink" href="#getti
 </code></pre></div>
 <h2 id="loops">Loops<a class="headerlink" href="#loops" title="Permanent link">&para;</a></h2>
 <h3 id="for-loop">For Loop<a class="headerlink" href="#for-loop" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code><span class="k">for</span> <span class="p">(</span><span class="kt">int</span> <span class="n">i</span><span class="o">=</span><span class="mi">0</span><span class="p">;</span> <span class="n">i</span><span class="o">&lt;</span><span class="mi">10</span><span class="p">;</span> <span class="n">i</span><span class="o">+=</span><span class="mi">1</span><span class="p">)</span>
+<div class="highlight"><pre><span></span><code><span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="kt">int</span><span class="w"> </span><span class="n">i</span><span class="o">=</span><span class="mi">0</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="o">&lt;</span><span class="mi">10</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="o">+=</span><span class="mi">1</span><span class="p">)</span>
 <span class="p">{</span>
-    <span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Value of i is: &quot;</span> <span class="o">&lt;&lt;</span> <span class="n">i</span> <span class="o">&lt;&lt;</span> <span class="n">endl</span><span class="p">;</span>
+<span class="w">    </span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Value of i is: &quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">endl</span><span class="p">;</span>
 <span class="p">}</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Value of i is: 0
@@ -3521,12 +3661,12 @@ <h3 id="for-loop">For Loop<a class="headerlink" href="#for-loop" title="Permanen
 Value of i is: 9
 </code></pre></div>
 <h3 id="while-loop">While Loop<a class="headerlink" href="#while-loop" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code><span class="kt">int</span> <span class="n">idxWhile</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span>
+<div class="highlight"><pre><span></span><code><span class="kt">int</span><span class="w"> </span><span class="n">idxWhile</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">;</span>
 
-<span class="k">while</span> <span class="p">(</span><span class="n">idxWhile</span> <span class="o">&lt;</span> <span class="mi">10</span><span class="p">)</span>
+<span class="k">while</span><span class="w"> </span><span class="p">(</span><span class="n">idxWhile</span><span class="w"> </span><span class="o">&lt;</span><span class="w"> </span><span class="mi">10</span><span class="p">)</span>
 <span class="p">{</span>
-    <span class="n">idxWhile</span> <span class="o">+=</span> <span class="mi">1</span><span class="p">;</span>
-    <span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Value of while loop i is: &quot;</span> <span class="o">&lt;&lt;</span> <span class="n">idxWhile</span> <span class="o">&lt;&lt;</span> <span class="n">endl</span><span class="p">;</span>
+<span class="w">    </span><span class="n">idxWhile</span><span class="w"> </span><span class="o">+=</span><span class="w"> </span><span class="mi">1</span><span class="p">;</span>
+<span class="w">    </span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Value of while loop i is: &quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">idxWhile</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">endl</span><span class="p">;</span>
 <span class="p">}</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Value of while loop i is: 1
@@ -3541,18 +3681,18 @@ <h3 id="while-loop">While Loop<a class="headerlink" href="#while-loop" title="Pe
 Value of while loop i is: 10
 </code></pre></div>
 <h3 id="while-loop-with-continuebreak">While Loop with Continue/Break<a class="headerlink" href="#while-loop-with-continuebreak" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code><span class="kt">int</span> <span class="n">idxWhileNew</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span>
+<div class="highlight"><pre><span></span><code><span class="kt">int</span><span class="w"> </span><span class="n">idxWhileNew</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">;</span>
 
-<span class="k">while</span> <span class="p">(</span><span class="n">idxWhileNew</span> <span class="o">&lt;</span> <span class="mi">100</span><span class="p">)</span>
+<span class="k">while</span><span class="w"> </span><span class="p">(</span><span class="n">idxWhileNew</span><span class="w"> </span><span class="o">&lt;</span><span class="w"> </span><span class="mi">100</span><span class="p">)</span>
 <span class="p">{</span>
-    <span class="n">idxWhileNew</span> <span class="o">+=</span> <span class="mi">1</span><span class="p">;</span>
-    <span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Value of while loop i is: &quot;</span> <span class="o">&lt;&lt;</span> <span class="n">idxWhile</span> <span class="o">&lt;&lt;</span> <span class="n">endl</span><span class="p">;</span>
-
-    <span class="k">if</span> <span class="p">(</span><span class="n">idxWhileNew</span> <span class="o">==</span> <span class="mi">10</span><span class="p">)</span>
-    <span class="p">{</span>
-        <span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">&quot;Max value of 10 reached!&quot;</span> <span class="o">&lt;&lt;</span> <span class="n">endl</span><span class="p">;</span>
-        <span class="k">break</span><span class="p">;</span>
-    <span class="p">}</span>
+<span class="w">    </span><span class="n">idxWhileNew</span><span class="w"> </span><span class="o">+=</span><span class="w"> </span><span class="mi">1</span><span class="p">;</span>
+<span class="w">    </span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Value of while loop i is: &quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">idxWhile</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">endl</span><span class="p">;</span>
+
+<span class="w">    </span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="n">idxWhileNew</span><span class="w"> </span><span class="o">==</span><span class="w"> </span><span class="mi">10</span><span class="p">)</span>
+<span class="w">    </span><span class="p">{</span>
+<span class="w">        </span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="s">&quot;Max value of 10 reached!&quot;</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">endl</span><span class="p">;</span>
+<span class="w">        </span><span class="k">break</span><span class="p">;</span>
+<span class="w">    </span><span class="p">}</span>
 <span class="p">}</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>Value of while loop i is: 10
@@ -3632,7 +3772,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -3695,12 +3835,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/programming/electron/electron/index.html b/programming/electron/electron/index.html
index 30f39b5dd..98f9184a3 100644
--- a/programming/electron/electron/index.html
+++ b/programming/electron/electron/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/programming/electron/electron/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1010,14 +1150,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" checked>
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1344,14 +1484,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1367,14 +1507,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1427,14 +1567,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1811,18 +1951,18 @@ <h1 id="electron">Electron<a class="headerlink" href="#electron" title="Permanen
 <h2 id="why-electron">Why Electron<a class="headerlink" href="#why-electron" title="Permanent link">&para;</a></h2>
 <p>We choose to cover electron as you can easily use it as a front-end application across any platforms (Windows, MacOS, Linux or even a mobile application) for your AI applications.</p>
 <h2 id="installation-of-electron">Installation of Electron<a class="headerlink" href="#installation-of-electron" title="Permanent link">&para;</a></h2>
-<div class="highlight"><pre><span></span><code>npm i -D electron@latest
+<div class="highlight"><pre><span></span><code>npm<span class="w"> </span>i<span class="w"> </span>-D<span class="w"> </span>electron@latest
 </code></pre></div>
 <h2 id="creating-electron-project">Creating Electron Project<a class="headerlink" href="#creating-electron-project" title="Permanent link">&para;</a></h2>
 <h3 id="critical-files">Critical Files<a class="headerlink" href="#critical-files" title="Permanent link">&para;</a></h3>
 <p>You should have 3 base files <code>package.json</code>, <code>main.js</code> and <code>index.html</code> to have a basic application.</p>
-<div class="highlight"><pre><span></span><code>mkdir app
-<span class="nb">cd</span> app
+<div class="highlight"><pre><span></span><code>mkdir<span class="w"> </span>app
+<span class="nb">cd</span><span class="w"> </span>app
 
-npm init
+npm<span class="w"> </span>init
 
-touch main.js
-touch index.html
+touch<span class="w"> </span>main.js
+touch<span class="w"> </span>index.html
 </code></pre></div>
 <h3 id="edit-packagejson">Edit package.json<a class="headerlink" href="#edit-packagejson" title="Permanent link">&para;</a></h3>
 <ul>
@@ -1837,70 +1977,70 @@ <h3 id="edit-packagejson">Edit package.json<a class="headerlink" href="#edit-pac
 </li>
 </ul>
 <div class="highlight"><pre><span></span><code><span class="p">{</span>
-  <span class="nt">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;dlw&quot;</span><span class="p">,</span>
-  <span class="nt">&quot;version&quot;</span><span class="p">:</span> <span class="s2">&quot;0.1.0&quot;</span><span class="p">,</span>
-  <span class="nt">&quot;main&quot;</span><span class="p">:</span> <span class="s2">&quot;main.js&quot;</span><span class="p">,</span>
-  <span class="nt">&quot;scripts&quot;</span><span class="p">:</span> <span class="p">{</span>
-    <span class="nt">&quot;start&quot;</span><span class="p">:</span> <span class="s2">&quot;electron .&quot;</span>
-  <span class="p">},</span>
-  <span class="nt">&quot;devDependencies&quot;</span><span class="p">:</span> <span class="p">{</span>
-    <span class="nt">&quot;electron&quot;</span><span class="p">:</span> <span class="s2">&quot;^6.0.8&quot;</span>
-  <span class="p">}</span>
+<span class="w">  </span><span class="nt">&quot;name&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;dlw&quot;</span><span class="p">,</span>
+<span class="w">  </span><span class="nt">&quot;version&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;0.1.0&quot;</span><span class="p">,</span>
+<span class="w">  </span><span class="nt">&quot;main&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;main.js&quot;</span><span class="p">,</span>
+<span class="w">  </span><span class="nt">&quot;scripts&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
+<span class="w">    </span><span class="nt">&quot;start&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;electron .&quot;</span>
+<span class="w">  </span><span class="p">},</span>
+<span class="w">  </span><span class="nt">&quot;devDependencies&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
+<span class="w">    </span><span class="nt">&quot;electron&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;^6.0.8&quot;</span>
+<span class="w">  </span><span class="p">}</span>
 <span class="p">}</span>
 </code></pre></div>
 <h3 id="edit-mainjs">Edit main.js<a class="headerlink" href="#edit-mainjs" title="Permanent link">&para;</a></h3>
 <p>This beautiful boilerplate code is provided by Electron, full credits to them. </p>
-<div class="highlight"><pre><span></span><code><span class="kd">const</span> <span class="p">{</span> <span class="nx">app</span><span class="p">,</span> <span class="nx">BrowserWindow</span> <span class="p">}</span> <span class="o">=</span> <span class="nx">require</span><span class="p">(</span><span class="s1">&#39;electron&#39;</span><span class="p">)</span>
+<div class="highlight"><pre><span></span><code><span class="kd">const</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="nx">app</span><span class="p">,</span><span class="w"> </span><span class="nx">BrowserWindow</span><span class="w"> </span><span class="p">}</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nx">require</span><span class="p">(</span><span class="s1">&#39;electron&#39;</span><span class="p">)</span>
 
 <span class="c1">// Keep a global reference of the window object, if you don&#39;t, the window will</span>
 <span class="c1">// be closed automatically when the JavaScript object is garbage collected.</span>
-<span class="kd">let</span> <span class="nx">win</span>
-
-<span class="kd">function</span> <span class="nx">createWindow</span> <span class="p">()</span> <span class="p">{</span>
-  <span class="c1">// Create the browser window.</span>
-  <span class="nx">win</span> <span class="o">=</span> <span class="k">new</span> <span class="nx">BrowserWindow</span><span class="p">({</span>
-    <span class="nx">width</span><span class="o">:</span> <span class="mf">800</span><span class="p">,</span>
-    <span class="nx">height</span><span class="o">:</span> <span class="mf">600</span><span class="p">,</span>
-    <span class="nx">webPreferences</span><span class="o">:</span> <span class="p">{</span>
-      <span class="nx">nodeIntegration</span><span class="o">:</span> <span class="kc">true</span>
-    <span class="p">}</span>
-  <span class="p">})</span>
-
-  <span class="c1">// and load the index.html of the app.</span>
-  <span class="nx">win</span><span class="p">.</span><span class="nx">loadFile</span><span class="p">(</span><span class="s1">&#39;index.html&#39;</span><span class="p">)</span>
-
-  <span class="c1">// Open the DevTools.</span>
-  <span class="c1">// win.webContents.openDevTools()</span>
-
-  <span class="c1">// Emitted when the window is closed.</span>
-  <span class="nx">win</span><span class="p">.</span><span class="nx">on</span><span class="p">(</span><span class="s1">&#39;closed&#39;</span><span class="p">,</span> <span class="p">()</span> <span class="p">=&gt;</span> <span class="p">{</span>
-    <span class="c1">// Dereference the window object, usually you would store windows</span>
-    <span class="c1">// in an array if your app supports multi windows, this is the time</span>
-    <span class="c1">// when you should delete the corresponding element.</span>
-    <span class="nx">win</span> <span class="o">=</span> <span class="kc">null</span>
-  <span class="p">})</span>
+<span class="kd">let</span><span class="w"> </span><span class="nx">win</span>
+
+<span class="kd">function</span><span class="w"> </span><span class="nx">createWindow</span><span class="w"> </span><span class="p">()</span><span class="w"> </span><span class="p">{</span>
+<span class="w">  </span><span class="c1">// Create the browser window.</span>
+<span class="w">  </span><span class="nx">win</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ow">new</span><span class="w"> </span><span class="nx">BrowserWindow</span><span class="p">({</span>
+<span class="w">    </span><span class="nx">width</span><span class="o">:</span><span class="w"> </span><span class="mf">800</span><span class="p">,</span>
+<span class="w">    </span><span class="nx">height</span><span class="o">:</span><span class="w"> </span><span class="mf">600</span><span class="p">,</span>
+<span class="w">    </span><span class="nx">webPreferences</span><span class="o">:</span><span class="w"> </span><span class="p">{</span>
+<span class="w">      </span><span class="nx">nodeIntegration</span><span class="o">:</span><span class="w"> </span><span class="kc">true</span>
+<span class="w">    </span><span class="p">}</span>
+<span class="w">  </span><span class="p">})</span>
+
+<span class="w">  </span><span class="c1">// and load the index.html of the app.</span>
+<span class="w">  </span><span class="nx">win</span><span class="p">.</span><span class="nx">loadFile</span><span class="p">(</span><span class="s1">&#39;index.html&#39;</span><span class="p">)</span>
+
+<span class="w">  </span><span class="c1">// Open the DevTools.</span>
+<span class="w">  </span><span class="c1">// win.webContents.openDevTools()</span>
+
+<span class="w">  </span><span class="c1">// Emitted when the window is closed.</span>
+<span class="w">  </span><span class="nx">win</span><span class="p">.</span><span class="nx">on</span><span class="p">(</span><span class="s1">&#39;closed&#39;</span><span class="p">,</span><span class="w"> </span><span class="p">()</span><span class="w"> </span><span class="p">=&gt;</span><span class="w"> </span><span class="p">{</span>
+<span class="w">    </span><span class="c1">// Dereference the window object, usually you would store windows</span>
+<span class="w">    </span><span class="c1">// in an array if your app supports multi windows, this is the time</span>
+<span class="w">    </span><span class="c1">// when you should delete the corresponding element.</span>
+<span class="w">    </span><span class="nx">win</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="kc">null</span>
+<span class="w">  </span><span class="p">})</span>
 <span class="p">}</span>
 
 <span class="c1">// This method will be called when Electron has finished</span>
 <span class="c1">// initialization and is ready to create browser windows.</span>
 <span class="c1">// Some APIs can only be used after this event occurs.</span>
-<span class="nx">app</span><span class="p">.</span><span class="nx">on</span><span class="p">(</span><span class="s1">&#39;ready&#39;</span><span class="p">,</span> <span class="nx">createWindow</span><span class="p">)</span>
+<span class="nx">app</span><span class="p">.</span><span class="nx">on</span><span class="p">(</span><span class="s1">&#39;ready&#39;</span><span class="p">,</span><span class="w"> </span><span class="nx">createWindow</span><span class="p">)</span>
 
 <span class="c1">// Quit when all windows are closed.</span>
-<span class="nx">app</span><span class="p">.</span><span class="nx">on</span><span class="p">(</span><span class="s1">&#39;window-all-closed&#39;</span><span class="p">,</span> <span class="p">()</span> <span class="p">=&gt;</span> <span class="p">{</span>
-  <span class="c1">// On macOS it is common for applications and their menu bar</span>
-  <span class="c1">// to stay active until the user quits explicitly with Cmd + Q</span>
-  <span class="k">if</span> <span class="p">(</span><span class="nx">process</span><span class="p">.</span><span class="nx">platform</span> <span class="o">!==</span> <span class="s1">&#39;darwin&#39;</span><span class="p">)</span> <span class="p">{</span>
-    <span class="nx">app</span><span class="p">.</span><span class="nx">quit</span><span class="p">()</span>
-  <span class="p">}</span>
+<span class="nx">app</span><span class="p">.</span><span class="nx">on</span><span class="p">(</span><span class="s1">&#39;window-all-closed&#39;</span><span class="p">,</span><span class="w"> </span><span class="p">()</span><span class="w"> </span><span class="p">=&gt;</span><span class="w"> </span><span class="p">{</span>
+<span class="w">  </span><span class="c1">// On macOS it is common for applications and their menu bar</span>
+<span class="w">  </span><span class="c1">// to stay active until the user quits explicitly with Cmd + Q</span>
+<span class="w">  </span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="nx">process</span><span class="p">.</span><span class="nx">platform</span><span class="w"> </span><span class="o">!==</span><span class="w"> </span><span class="s1">&#39;darwin&#39;</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
+<span class="w">    </span><span class="nx">app</span><span class="p">.</span><span class="nx">quit</span><span class="p">()</span>
+<span class="w">  </span><span class="p">}</span>
 <span class="p">})</span>
 
-<span class="nx">app</span><span class="p">.</span><span class="nx">on</span><span class="p">(</span><span class="s1">&#39;activate&#39;</span><span class="p">,</span> <span class="p">()</span> <span class="p">=&gt;</span> <span class="p">{</span>
-  <span class="c1">// On macOS it&#39;s common to re-create a window in the app when the</span>
-  <span class="c1">// dock icon is clicked and there are no other windows open.</span>
-  <span class="k">if</span> <span class="p">(</span><span class="nx">win</span> <span class="o">===</span> <span class="kc">null</span><span class="p">)</span> <span class="p">{</span>
-    <span class="nx">createWindow</span><span class="p">()</span>
-  <span class="p">}</span>
+<span class="nx">app</span><span class="p">.</span><span class="nx">on</span><span class="p">(</span><span class="s1">&#39;activate&#39;</span><span class="p">,</span><span class="w"> </span><span class="p">()</span><span class="w"> </span><span class="p">=&gt;</span><span class="w"> </span><span class="p">{</span>
+<span class="w">  </span><span class="c1">// On macOS it&#39;s common to re-create a window in the app when the</span>
+<span class="w">  </span><span class="c1">// dock icon is clicked and there are no other windows open.</span>
+<span class="w">  </span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="nx">win</span><span class="w"> </span><span class="o">===</span><span class="w"> </span><span class="kc">null</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
+<span class="w">    </span><span class="nx">createWindow</span><span class="p">()</span>
+<span class="w">  </span><span class="p">}</span>
 <span class="p">})</span>
 
 <span class="c1">// In this file you can include the rest of your app&#39;s specific main process</span>
@@ -1933,7 +2073,7 @@ <h3 id="edit-indexhtml">Edit index.html<a class="headerlink" href="#edit-indexht
 </code></pre></div>
 <h2 id="starting-app">Starting App<a class="headerlink" href="#starting-app" title="Permanent link">&para;</a></h2>
 <p>This will start your electron application.</p>
-<div class="highlight"><pre><span></span><code>npm start
+<div class="highlight"><pre><span></span><code>npm<span class="w"> </span>start
 </code></pre></div>
 <h2 id="packaging-electron-app">Packaging Electron App<a class="headerlink" href="#packaging-electron-app" title="Permanent link">&para;</a></h2>
 <h3 id="wine">Wine<a class="headerlink" href="#wine" title="Permanent link">&para;</a></h3>
@@ -1941,51 +2081,51 @@ <h3 id="wine">Wine<a class="headerlink" href="#wine" title="Permanent link">&par
 <h4 id="installation-of-wine">Installation of Wine<a class="headerlink" href="#installation-of-wine" title="Permanent link">&para;</a></h4>
 <p>This assumes installation on Ubuntu 16.04 <code>xenial</code>, if you're on Ubuntu 18.04 or 19.04, change to <code>bionic</code> and <code>disco</code> respectively.</p>
 <p>Also, this works on 64-bit system architecture.</p>
-<div class="highlight"><pre><span></span><code><span class="nb">cd</span> ~
-wget -qO - https://dl.winehq.org/wine-builds/winehq.key <span class="p">|</span> sudo apt-key add -
-sudo apt-add-repository <span class="s1">&#39;deb https://dl.winehq.org/wine-builds/ubuntu/ xenial main&#39;</span>
-sudo apt-get update
-sudo apt-get install --install-recommends winehq-stable
+<div class="highlight"><pre><span></span><code><span class="nb">cd</span><span class="w"> </span>~
+wget<span class="w"> </span>-qO<span class="w"> </span>-<span class="w"> </span>https://dl.winehq.org/wine-builds/winehq.key<span class="w"> </span><span class="p">|</span><span class="w"> </span>sudo<span class="w"> </span>apt-key<span class="w"> </span>add<span class="w"> </span>-
+sudo<span class="w"> </span>apt-add-repository<span class="w"> </span><span class="s1">&#39;deb https://dl.winehq.org/wine-builds/ubuntu/ xenial main&#39;</span>
+sudo<span class="w"> </span>apt-get<span class="w"> </span>update
+sudo<span class="w"> </span>apt-get<span class="w"> </span>install<span class="w"> </span>--install-recommends<span class="w"> </span>winehq-stable
 
-sudo chown root:root ~/.wine
+sudo<span class="w"> </span>chown<span class="w"> </span>root:root<span class="w"> </span>~/.wine
 </code></pre></div>
 <h4 id="check-wine-version">Check Wine Version<a class="headerlink" href="#check-wine-version" title="Permanent link">&para;</a></h4>
-<div class="highlight"><pre><span></span><code>wine --version
+<div class="highlight"><pre><span></span><code>wine<span class="w"> </span>--version
 </code></pre></div>
 <h3 id="packaging-windows-application">Packaging Windows Application<a class="headerlink" href="#packaging-windows-application" title="Permanent link">&para;</a></h3>
 <p>This packages the application churning the necessary files and the executable <code>app.exe</code> for windows 64 bit.</p>
-<div class="highlight"><pre><span></span><code>electron-packager ./app app --platform<span class="o">=</span>win32 --arch<span class="o">=</span>x64
+<div class="highlight"><pre><span></span><code>electron-packager<span class="w"> </span>./app<span class="w"> </span>app<span class="w"> </span>--platform<span class="o">=</span>win32<span class="w"> </span>--arch<span class="o">=</span>x64
 </code></pre></div>
 <h2 id="python-scripts">Python Scripts<a class="headerlink" href="#python-scripts" title="Permanent link">&para;</a></h2>
 <h3 id="installing-python-node-package">Installing Python Node Package<a class="headerlink" href="#installing-python-node-package" title="Permanent link">&para;</a></h3>
 <p>So we want to easily create Python scripts and run through Javascript in the Electron application. This can be done via <code>python-shell</code> npm package.</p>
-<div class="highlight"><pre><span></span><code>sudo npm install --save python-shell 
+<div class="highlight"><pre><span></span><code>sudo<span class="w"> </span>npm<span class="w"> </span>install<span class="w"> </span>--save<span class="w"> </span>python-shell<span class="w"> </span>
 </code></pre></div>
 <h3 id="creating-hello-from-js">Creating "Hello from JS"<a class="headerlink" href="#creating-hello-from-js" title="Permanent link">&para;</a></h3>
 <h4 id="javascript">Javascript<a class="headerlink" href="#javascript" title="Permanent link">&para;</a></h4>
 <p>In your <code>main.js</code> file, you would want to add the following code. </p>
 <p>This leverages on the <code>python-shell</code> package to send a message to <code>hello_world.py</code> and receive the message subsequently.</p>
 <div class="highlight"><pre><span></span><code><span class="c1">// Start Python shell</span>
-<span class="kd">let</span> <span class="p">{</span><span class="nx">PythonShell</span><span class="p">}</span> <span class="o">=</span> <span class="nx">require</span><span class="p">(</span><span class="s1">&#39;python-shell&#39;</span><span class="p">)</span>
+<span class="kd">let</span><span class="w"> </span><span class="p">{</span><span class="nx">PythonShell</span><span class="p">}</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nx">require</span><span class="p">(</span><span class="s1">&#39;python-shell&#39;</span><span class="p">)</span>
 
 <span class="c1">// Start shell for specific script for communicating</span>
-<span class="kd">let</span> <span class="nx">pyshell</span> <span class="o">=</span> <span class="k">new</span> <span class="nx">PythonShell</span><span class="p">(</span><span class="s1">&#39;./scripts/hello_world.py&#39;</span><span class="p">);</span>
+<span class="kd">let</span><span class="w"> </span><span class="nx">pyshell</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ow">new</span><span class="w"> </span><span class="nx">PythonShell</span><span class="p">(</span><span class="s1">&#39;./scripts/hello_world.py&#39;</span><span class="p">);</span>
 
 <span class="c1">// Send a message to the Python script via stdin</span>
 <span class="nx">pyshell</span><span class="p">.</span><span class="nx">send</span><span class="p">(</span><span class="s1">&#39;Hello from JS&#39;</span><span class="p">);</span>
 
 <span class="c1">// Receive message from Python script</span>
-<span class="nx">pyshell</span><span class="p">.</span><span class="nx">on</span><span class="p">(</span><span class="s1">&#39;message&#39;</span><span class="p">,</span> <span class="kd">function</span> <span class="p">(</span><span class="nx">message</span><span class="p">)</span> <span class="p">{</span>
-  <span class="nx">console</span><span class="p">.</span><span class="nx">log</span><span class="p">(</span><span class="nx">message</span><span class="p">);</span>
+<span class="nx">pyshell</span><span class="p">.</span><span class="nx">on</span><span class="p">(</span><span class="s1">&#39;message&#39;</span><span class="p">,</span><span class="w"> </span><span class="kd">function</span><span class="w"> </span><span class="p">(</span><span class="nx">message</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
+<span class="w">  </span><span class="nx">console</span><span class="p">.</span><span class="nx">log</span><span class="p">(</span><span class="nx">message</span><span class="p">);</span>
 
 <span class="p">});</span>
 
 <span class="c1">// End the input stream and allow the process to exit</span>
-<span class="nx">pyshell</span><span class="p">.</span><span class="nx">end</span><span class="p">(</span><span class="kd">function</span> <span class="p">(</span><span class="nx">err</span><span class="p">,</span> <span class="nx">code</span><span class="p">,</span> <span class="nx">signal</span><span class="p">)</span> <span class="p">{</span>
-  <span class="k">if</span> <span class="p">(</span><span class="nx">err</span><span class="p">)</span> <span class="k">throw</span> <span class="nx">err</span><span class="p">;</span>
+<span class="nx">pyshell</span><span class="p">.</span><span class="nx">end</span><span class="p">(</span><span class="kd">function</span><span class="w"> </span><span class="p">(</span><span class="nx">err</span><span class="p">,</span><span class="w"> </span><span class="nx">code</span><span class="p">,</span><span class="w"> </span><span class="nx">signal</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
+<span class="w">  </span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="nx">err</span><span class="p">)</span><span class="w"> </span><span class="k">throw</span><span class="w"> </span><span class="nx">err</span><span class="p">;</span>
 <span class="c1">//  console.log(&#39;The exit code was: &#39; + code);</span>
 <span class="c1">//  console.log(&#39;The exit signal was: &#39; + signal);</span>
-  <span class="nx">console</span><span class="p">.</span><span class="nx">log</span><span class="p">(</span><span class="s1">&#39;finished&#39;</span><span class="p">);</span>
+<span class="w">  </span><span class="nx">console</span><span class="p">.</span><span class="nx">log</span><span class="p">(</span><span class="s1">&#39;finished&#39;</span><span class="p">);</span>
 <span class="p">});</span>
 </code></pre></div>
 <h4 id="python">Python<a class="headerlink" href="#python" title="Permanent link">&para;</a></h4>
@@ -1998,7 +2138,7 @@ <h4 id="python">Python<a class="headerlink" href="#python" title="Permanent link
 </code></pre></div>
 <h4 id="run-app">Run App<a class="headerlink" href="#run-app" title="Permanent link">&para;</a></h4>
 <p>Run via <code>npm start</code> and you'll see this in your bash output. Viola! We managed to call <code>hello_world.py</code> via <code>main.js</code> through the <code>python-shell</code> package. Next task, we will be passing this message to <code>index.html</code>.</p>
-<div class="highlight"><pre><span></span><code>Hello from JS
+<div class="highlight"><pre><span></span><code>Hello<span class="w"> </span>from<span class="w"> </span>JS
 
 finished
 </code></pre></div>
@@ -2067,7 +2207,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -2130,12 +2270,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/programming/intro/index.html b/programming/intro/index.html
index 75ab1d234..d4f1bf2fb 100644
--- a/programming/intro/index.html
+++ b/programming/intro/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/programming/intro/">
       
       <link rel="icon" href="../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1010,14 +1150,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" checked>
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1158,14 +1298,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1181,14 +1321,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1241,14 +1381,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1452,7 +1592,7 @@
                 
                 <h1 id="programming">Programming<a class="headerlink" href="#programming" title="Permanent link">&para;</a></h1>
 <p>We'll be covering C++, Python, Bash and more for end-to-end AI deployments.</p>
-<div class="tabbed-set" data-tabs="1:1"><input checked="checked" id="__tabbed_1_1" name="__tabbed_1" type="radio" /><label for="__tabbed_1_1">Packages and Languages you will learn to use</label><div class="tabbed-content">
+<div class="tabbed-set" data-tabs="1:1"><input checked="checked" id="programming-packages-and-languages-you-will-learn-to-use" name="__tabbed_1" type="radio" /><label for="programming-packages-and-languages-you-will-learn-to-use">Packages and Languages you will learn to use</label><div class="tabbed-content">
 <ul>
 <li>C++</li>
 <li>Python</li>
@@ -1532,7 +1672,7 @@ <h1 id="programming">Programming<a class="headerlink" href="#programming" title=
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1595,12 +1735,16 @@ <h1 id="programming">Programming<a class="headerlink" href="#programming" title=
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/programming/javascript/javascript/index.html b/programming/javascript/javascript/index.html
index ff21bfbe1..777fefc7d 100644
--- a/programming/javascript/javascript/index.html
+++ b/programming/javascript/javascript/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/programming/javascript/javascript/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1010,14 +1150,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" checked>
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1269,14 +1409,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1292,14 +1432,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1352,14 +1492,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1675,7 +1815,7 @@ <h2 id="installation-of-ijavascript">Installation of iJavascript<a class="header
 <h2 id="variables-constants">Variables &amp; Constants<a class="headerlink" href="#variables-constants" title="Permanent link">&para;</a></h2>
 <h3 id="variable">Variable<a class="headerlink" href="#variable" title="Permanent link">&para;</a></h3>
 <h4 id="declaring-variable">Declaring Variable<a class="headerlink" href="#declaring-variable" title="Permanent link">&para;</a></h4>
-<div class="highlight"><pre><span></span><code><span class="kd">var</span> <span class="nx">randomNumber</span> <span class="o">=</span> <span class="mf">1.52</span>
+<div class="highlight"><pre><span></span><code><span class="kd">var</span><span class="w"> </span><span class="nx">randomNumber</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mf">1.52</span>
 </code></pre></div>
 <h4 id="read-out-variable">Read out Variable<a class="headerlink" href="#read-out-variable" title="Permanent link">&para;</a></h4>
 <div class="highlight"><pre><span></span><code><span class="c1">// Check value</span>
@@ -1696,7 +1836,7 @@ <h4 id="read-out-via-console">Read out via Console<a class="headerlink" href="#r
 <h3 id="constants">Constants<a class="headerlink" href="#constants" title="Permanent link">&para;</a></h3>
 <h3 id="declaring-constant">Declaring Constant<a class="headerlink" href="#declaring-constant" title="Permanent link">&para;</a></h3>
 <p>This is a weird one, because in Javascript when you declare a constant, you are essentially unable to overwrite the constant subsequently. It's useful when you want a fixed value.</p>
-<div class="highlight"><pre><span></span><code><span class="kd">const</span> <span class="nx">cannotOverwriteNumber</span> <span class="o">=</span> <span class="mf">2.22</span>
+<div class="highlight"><pre><span></span><code><span class="kd">const</span><span class="w"> </span><span class="nx">cannotOverwriteNumber</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mf">2.22</span>
 </code></pre></div>
 <h4 id="read-out-constant">Read out Constant<a class="headerlink" href="#read-out-constant" title="Permanent link">&para;</a></h4>
 <div class="highlight"><pre><span></span><code><span class="nx">cannotOverwriteNumber</span>
@@ -1705,7 +1845,7 @@ <h4 id="read-out-constant">Read out Constant<a class="headerlink" href="#read-ou
 </code></pre></div>
 <h4 id="overwrite-constant-error">Overwrite Constant (Error)<a class="headerlink" href="#overwrite-constant-error" title="Permanent link">&para;</a></h4>
 <p>This will throw an error because this constant has been declared once and you cannot do it again.</p>
-<div class="highlight"><pre><span></span><code><span class="kd">const</span> <span class="nx">cannotOverwriteNumber</span> <span class="o">=</span> <span class="mf">1.52</span>
+<div class="highlight"><pre><span></span><code><span class="kd">const</span><span class="w"> </span><span class="nx">cannotOverwriteNumber</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mf">1.52</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>evalmachine.&lt;anonymous&gt;:1
 
@@ -1802,7 +1942,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1865,12 +2005,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/programming/numpycupy/linalg_numpy_cupy/index.html b/programming/numpycupy/linalg_numpy_cupy/index.html
index 4aecfba22..b46bacd27 100644
--- a/programming/numpycupy/linalg_numpy_cupy/index.html
+++ b/programming/numpycupy/linalg_numpy_cupy/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/programming/numpycupy/linalg_numpy_cupy/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1010,14 +1150,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" checked>
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1316,14 +1456,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1339,14 +1479,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1399,14 +1539,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1952,7 +2092,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -2015,12 +2155,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/programming/plotting/intro/index.html b/programming/plotting/intro/index.html
index 15d8a2025..046c5ac15 100644
--- a/programming/plotting/intro/index.html
+++ b/programming/plotting/intro/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/programming/plotting/intro/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -355,7 +371,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -914,11 +930,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -946,14 +1086,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1006,14 +1146,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1144,14 +1284,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1167,14 +1307,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1227,14 +1367,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1437,7 +1577,7 @@
                 
                 
                 <h1 id="plotting">Plotting<a class="headerlink" href="#plotting" title="Permanent link">&para;</a></h1>
-<div class="tabbed-set" data-tabs="1:1"><input checked="checked" id="__tabbed_1_1" name="__tabbed_1" type="radio" /><label for="__tabbed_1_1">Packages you will learn to use</label><div class="tabbed-content">
+<div class="tabbed-set" data-tabs="1:1"><input checked="checked" id="plotting-packages-you-will-learn-to-use" name="__tabbed_1" type="radio" /><label for="plotting-packages-you-will-learn-to-use">Packages you will learn to use</label><div class="tabbed-content">
 <ol>
 <li>Fast professional static plots: ggplot2 and rpy2</li>
 <li>Complicated static plots: matplotlib</li>
@@ -1479,7 +1619,7 @@ <h1 id="plotting">Plotting<a class="headerlink" href="#plotting" title="Permanen
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1542,12 +1682,16 @@ <h1 id="plotting">Plotting<a class="headerlink" href="#plotting" title="Permanen
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/programming/python/python/index.html b/programming/python/python/index.html
index 83fff93d8..0ef5d1944 100644
--- a/programming/python/python/index.html
+++ b/programming/python/python/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/programming/python/python/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1010,14 +1150,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" checked>
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1629,14 +1769,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1652,14 +1792,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1712,14 +1852,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -3152,7 +3292,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -3215,12 +3355,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/programming/r/r/index.html b/programming/r/r/index.html
index fac7bedf7..0a9380c0d 100644
--- a/programming/r/r/index.html
+++ b/programming/r/r/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/programming/r/r/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1010,14 +1150,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" checked>
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1257,14 +1397,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1280,14 +1420,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1340,14 +1480,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1636,12 +1776,12 @@
                 <h1 id="r">R<a class="headerlink" href="#r" title="Permanent link">&para;</a></h1>
 <h2 id="quick-start-basics">Quick Start Basics<a class="headerlink" href="#quick-start-basics" title="Permanent link">&para;</a></h2>
 <h3 id="variable-object">Variable Object<a class="headerlink" href="#variable-object" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code><span class="n">var_a</span> <span class="o">&lt;-</span> <span class="m">2</span>
+<div class="highlight"><pre><span></span><code><span class="n">var_a</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="m">2</span>
 <span class="nf">print</span><span class="p">(</span><span class="n">var_a</span><span class="p">)</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>[1] 2
 </code></pre></div>
-<div class="highlight"><pre><span></span><code><span class="n">var_b</span> <span class="o">=</span> <span class="m">3</span>
+<div class="highlight"><pre><span></span><code><span class="n">var_b</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="m">3</span>
 <span class="nf">print</span><span class="p">(</span><span class="n">var_b</span><span class="p">)</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>[1] 3
@@ -1652,7 +1792,7 @@ <h3 id="check-object-type">Check Object Type<a class="headerlink" href="#check-o
 </code></pre></div>
 <p>'numeric'</p>
 <h3 id="list-vector">List / Vector<a class="headerlink" href="#list-vector" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code><span class="n">list_a</span> <span class="o">&lt;-</span> <span class="nf">c</span><span class="p">(</span><span class="m">0</span><span class="p">,</span> <span class="m">1</span><span class="p">,</span> <span class="m">2</span><span class="p">)</span>
+<div class="highlight"><pre><span></span><code><span class="n">list_a</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">c</span><span class="p">(</span><span class="m">0</span><span class="p">,</span><span class="w"> </span><span class="m">1</span><span class="p">,</span><span class="w"> </span><span class="m">2</span><span class="p">)</span>
 <span class="n">list_a</span>
 </code></pre></div>
 <style>
@@ -1663,7 +1803,7 @@ <h3 id="list-vector">List / Vector<a class="headerlink" href="#list-vector" titl
 <ol class=list-inline><li>0</li><li>1</li><li>2</li></ol>
 
 <div class="highlight"><pre><span></span><code><span class="c1"># It&#39;s inclusive start till end, unlike Python which leaves the last out</span>
-<span class="n">list_b</span> <span class="o">&lt;-</span> <span class="nf">c</span><span class="p">(</span><span class="m">0</span><span class="o">:</span><span class="m">6</span><span class="p">)</span>
+<span class="n">list_b</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">c</span><span class="p">(</span><span class="m">0</span><span class="o">:</span><span class="m">6</span><span class="p">)</span>
 <span class="n">list_b</span>
 </code></pre></div>
 <style>
@@ -1674,7 +1814,7 @@ <h3 id="list-vector">List / Vector<a class="headerlink" href="#list-vector" titl
 <ol class=list-inline><li>0</li><li>1</li><li>2</li><li>3</li><li>4</li><li>5</li><li>6</li></ol>
 
 <div class="highlight"><pre><span></span><code><span class="c1"># Similar to a Python dictionary</span>
-<span class="n">list_c</span> <span class="o">&lt;-</span> <span class="nf">list</span><span class="p">(</span><span class="n">john_height</span><span class="o">=</span><span class="m">120</span><span class="p">,</span> <span class="n">doe_height</span><span class="o">=</span><span class="m">150</span><span class="p">)</span>
+<span class="n">list_c</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">list</span><span class="p">(</span><span class="n">john_height</span><span class="o">=</span><span class="m">120</span><span class="p">,</span><span class="w"> </span><span class="n">doe_height</span><span class="o">=</span><span class="m">150</span><span class="p">)</span>
 <span class="n">list_c</span>
 </code></pre></div>
 <dl>
@@ -1704,8 +1844,8 @@ <h3 id="list-vector">List / Vector<a class="headerlink" href="#list-vector" titl
 
 <h3 id="matrix">Matrix<a class="headerlink" href="#matrix" title="Permanent link">&para;</a></h3>
 <div class="highlight"><pre><span></span><code><span class="c1"># Create 2 vectors</span>
-<span class="n">vect_a</span> <span class="o">&lt;-</span> <span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span> <span class="m">2</span><span class="p">)</span>
-<span class="n">vect_b</span> <span class="o">&lt;-</span> <span class="nf">c</span><span class="p">(</span><span class="m">3</span><span class="p">,</span> <span class="m">4</span><span class="p">)</span>
+<span class="n">vect_a</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span><span class="w"> </span><span class="m">2</span><span class="p">)</span>
+<span class="n">vect_b</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">c</span><span class="p">(</span><span class="m">3</span><span class="p">,</span><span class="w"> </span><span class="m">4</span><span class="p">)</span>
 
 <span class="n">vect_a</span>
 <span class="n">vect_b</span>
@@ -1725,7 +1865,7 @@ <h3 id="matrix">Matrix<a class="headerlink" href="#matrix" title="Permanent link
 <ol class=list-inline><li>3</li><li>4</li></ol>
 
 <div class="highlight"><pre><span></span><code><span class="c1"># Bind as row</span>
-<span class="n">mat_c</span> <span class="o">&lt;-</span> <span class="nf">rbind</span><span class="p">(</span><span class="n">vect_a</span><span class="p">,</span> <span class="n">vect_b</span><span class="p">)</span>
+<span class="n">mat_c</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">rbind</span><span class="p">(</span><span class="n">vect_a</span><span class="p">,</span><span class="w"> </span><span class="n">vect_b</span><span class="p">)</span>
 <span class="n">mat_c</span>
 </code></pre></div>
 <table class="dataframe">
@@ -1737,7 +1877,7 @@ <h3 id="matrix">Matrix<a class="headerlink" href="#matrix" title="Permanent link
 </table>
 
 <div class="highlight"><pre><span></span><code><span class="c1"># Bind as column</span>
-<span class="n">mat_d</span> <span class="o">&lt;-</span> <span class="nf">cbind</span><span class="p">(</span><span class="n">vect_a</span><span class="p">,</span> <span class="n">vect_b</span><span class="p">)</span>
+<span class="n">mat_d</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">cbind</span><span class="p">(</span><span class="n">vect_a</span><span class="p">,</span><span class="w"> </span><span class="n">vect_b</span><span class="p">)</span>
 <span class="n">mat_d</span>
 </code></pre></div>
 <table class="dataframe">
@@ -1752,8 +1892,8 @@ <h3 id="matrix">Matrix<a class="headerlink" href="#matrix" title="Permanent link
 </table>
 
 <div class="highlight"><pre><span></span><code><span class="c1"># Create matrix</span>
-<span class="n">mat_e</span> <span class="o">&lt;-</span> <span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span> <span class="m">2</span><span class="p">,</span> <span class="m">3</span><span class="p">,</span> 
-                 <span class="m">4</span><span class="p">,</span> <span class="m">5</span><span class="p">,</span> <span class="m">6</span><span class="p">),</span> <span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span> <span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
+<span class="n">mat_e</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span><span class="w"> </span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="m">3</span><span class="p">,</span><span class="w"> </span>
+<span class="w">                 </span><span class="m">4</span><span class="p">,</span><span class="w"> </span><span class="m">5</span><span class="p">,</span><span class="w"> </span><span class="m">6</span><span class="p">),</span><span class="w"> </span><span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
 
 <span class="n">mat_e</span>
 </code></pre></div>
@@ -1766,7 +1906,7 @@ <h3 id="matrix">Matrix<a class="headerlink" href="#matrix" title="Permanent link
 </table>
 
 <div class="highlight"><pre><span></span><code><span class="c1"># Create matrix with byrow option</span>
-<span class="n">mat_g</span> <span class="o">&lt;-</span> <span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span> <span class="m">2</span><span class="p">,</span> <span class="m">3</span><span class="p">,</span> <span class="m">4</span><span class="p">,</span> <span class="m">5</span><span class="p">,</span> <span class="m">6</span><span class="p">),</span> <span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span> <span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">,</span> <span class="n">byrow</span><span class="o">=</span><span class="kc">FALSE</span><span class="p">)</span>
+<span class="n">mat_g</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span><span class="w"> </span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="m">3</span><span class="p">,</span><span class="w"> </span><span class="m">4</span><span class="p">,</span><span class="w"> </span><span class="m">5</span><span class="p">,</span><span class="w"> </span><span class="m">6</span><span class="p">),</span><span class="w"> </span><span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">,</span><span class="w"> </span><span class="n">byrow</span><span class="o">=</span><span class="kc">FALSE</span><span class="p">)</span>
 
 <span class="n">mat_g</span>
 </code></pre></div>
@@ -1779,7 +1919,7 @@ <h3 id="matrix">Matrix<a class="headerlink" href="#matrix" title="Permanent link
 </table>
 
 <div class="highlight"><pre><span></span><code><span class="c1"># Create same matrix with byrow option</span>
-<span class="n">mat_f</span> <span class="o">&lt;-</span> <span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span> <span class="m">2</span><span class="p">,</span> <span class="m">3</span><span class="p">,</span> <span class="m">4</span><span class="p">,</span> <span class="m">5</span><span class="p">,</span> <span class="m">6</span><span class="p">),</span> <span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span> <span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">,</span> <span class="n">byrow</span><span class="o">=</span><span class="kc">TRUE</span><span class="p">)</span>
+<span class="n">mat_f</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span><span class="w"> </span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="m">3</span><span class="p">,</span><span class="w"> </span><span class="m">4</span><span class="p">,</span><span class="w"> </span><span class="m">5</span><span class="p">,</span><span class="w"> </span><span class="m">6</span><span class="p">),</span><span class="w"> </span><span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">,</span><span class="w"> </span><span class="n">byrow</span><span class="o">=</span><span class="kc">TRUE</span><span class="p">)</span>
 
 <span class="n">mat_f</span>
 </code></pre></div>
@@ -1832,9 +1972,9 @@ <h3 id="matrix">Matrix<a class="headerlink" href="#matrix" title="Permanent link
 </table>
 
 <div class="highlight"><pre><span></span><code><span class="c1"># Add matrix</span>
-<span class="n">mat_1</span> <span class="o">&lt;-</span> <span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span> <span class="m">2</span><span class="p">,</span> <span class="m">3</span><span class="p">,</span> <span class="m">4</span><span class="p">,</span> <span class="m">5</span><span class="p">,</span> <span class="m">6</span><span class="p">),</span> <span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span> <span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
-<span class="n">mat_2</span> <span class="o">&lt;-</span> <span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span> <span class="m">2</span><span class="p">,</span> <span class="m">3</span><span class="p">,</span> <span class="m">4</span><span class="p">,</span> <span class="m">5</span><span class="p">,</span> <span class="m">6</span><span class="p">),</span> <span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span> <span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
-<span class="n">mat_new</span> <span class="o">&lt;-</span> <span class="n">mat_1</span> <span class="o">+</span> <span class="n">mat_2</span>
+<span class="n">mat_1</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span><span class="w"> </span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="m">3</span><span class="p">,</span><span class="w"> </span><span class="m">4</span><span class="p">,</span><span class="w"> </span><span class="m">5</span><span class="p">,</span><span class="w"> </span><span class="m">6</span><span class="p">),</span><span class="w"> </span><span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
+<span class="n">mat_2</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span><span class="w"> </span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="m">3</span><span class="p">,</span><span class="w"> </span><span class="m">4</span><span class="p">,</span><span class="w"> </span><span class="m">5</span><span class="p">,</span><span class="w"> </span><span class="m">6</span><span class="p">),</span><span class="w"> </span><span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
+<span class="n">mat_new</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="n">mat_1</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="n">mat_2</span>
 <span class="n">mat_1</span>
 <span class="n">mat_2</span>
 <span class="n">mat_new</span>
@@ -1864,9 +2004,9 @@ <h3 id="matrix">Matrix<a class="headerlink" href="#matrix" title="Permanent link
 </table>
 
 <div class="highlight"><pre><span></span><code><span class="c1"># Subtract matrix</span>
-<span class="n">mat_1</span> <span class="o">&lt;-</span> <span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span> <span class="m">2</span><span class="p">,</span> <span class="m">3</span><span class="p">,</span> <span class="m">4</span><span class="p">,</span> <span class="m">5</span><span class="p">,</span> <span class="m">6</span><span class="p">),</span> <span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span> <span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
-<span class="n">mat_2</span> <span class="o">&lt;-</span> <span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span> <span class="m">2</span><span class="p">,</span> <span class="m">3</span><span class="p">,</span> <span class="m">4</span><span class="p">,</span> <span class="m">5</span><span class="p">,</span> <span class="m">6</span><span class="p">),</span> <span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span> <span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
-<span class="n">mat_new</span> <span class="o">&lt;-</span> <span class="n">mat_1</span> <span class="o">-</span> <span class="n">mat_2</span>
+<span class="n">mat_1</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span><span class="w"> </span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="m">3</span><span class="p">,</span><span class="w"> </span><span class="m">4</span><span class="p">,</span><span class="w"> </span><span class="m">5</span><span class="p">,</span><span class="w"> </span><span class="m">6</span><span class="p">),</span><span class="w"> </span><span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
+<span class="n">mat_2</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span><span class="w"> </span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="m">3</span><span class="p">,</span><span class="w"> </span><span class="m">4</span><span class="p">,</span><span class="w"> </span><span class="m">5</span><span class="p">,</span><span class="w"> </span><span class="m">6</span><span class="p">),</span><span class="w"> </span><span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
+<span class="n">mat_new</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="n">mat_1</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="n">mat_2</span>
 <span class="n">mat_1</span>
 <span class="n">mat_2</span>
 <span class="n">mat_new</span>
@@ -1896,9 +2036,9 @@ <h3 id="matrix">Matrix<a class="headerlink" href="#matrix" title="Permanent link
 </table>
 
 <div class="highlight"><pre><span></span><code><span class="c1"># Element-wise multiplication / Hadamard product</span>
-<span class="n">mat_1</span> <span class="o">&lt;-</span> <span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span> <span class="m">2</span><span class="p">,</span> <span class="m">3</span><span class="p">,</span> <span class="m">4</span><span class="p">,</span> <span class="m">5</span><span class="p">,</span> <span class="m">6</span><span class="p">),</span> <span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span> <span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
-<span class="n">mat_2</span> <span class="o">&lt;-</span> <span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span> <span class="m">2</span><span class="p">,</span> <span class="m">3</span><span class="p">,</span> <span class="m">4</span><span class="p">,</span> <span class="m">5</span><span class="p">,</span> <span class="m">6</span><span class="p">),</span> <span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span> <span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
-<span class="n">mat_new</span> <span class="o">&lt;-</span> <span class="n">mat_1</span> <span class="o">*</span> <span class="n">mat_2</span>
+<span class="n">mat_1</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span><span class="w"> </span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="m">3</span><span class="p">,</span><span class="w"> </span><span class="m">4</span><span class="p">,</span><span class="w"> </span><span class="m">5</span><span class="p">,</span><span class="w"> </span><span class="m">6</span><span class="p">),</span><span class="w"> </span><span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
+<span class="n">mat_2</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span><span class="w"> </span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="m">3</span><span class="p">,</span><span class="w"> </span><span class="m">4</span><span class="p">,</span><span class="w"> </span><span class="m">5</span><span class="p">,</span><span class="w"> </span><span class="m">6</span><span class="p">),</span><span class="w"> </span><span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
+<span class="n">mat_new</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="n">mat_1</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="n">mat_2</span>
 <span class="n">mat_1</span>
 <span class="n">mat_2</span>
 <span class="n">mat_new</span>
@@ -1928,9 +2068,9 @@ <h3 id="matrix">Matrix<a class="headerlink" href="#matrix" title="Permanent link
 </table>
 
 <div class="highlight"><pre><span></span><code><span class="c1"># Matrix multiplication </span>
-<span class="n">mat_1</span> <span class="o">&lt;-</span> <span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span> <span class="m">2</span><span class="p">,</span> <span class="m">3</span><span class="p">,</span> <span class="m">4</span><span class="p">,</span> <span class="m">5</span><span class="p">,</span> <span class="m">6</span><span class="p">),</span> <span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span> <span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
-<span class="n">mat_2</span> <span class="o">&lt;-</span> <span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span> <span class="m">2</span><span class="p">,</span> <span class="m">3</span><span class="p">,</span> <span class="m">4</span><span class="p">,</span> <span class="m">5</span><span class="p">,</span> <span class="m">6</span><span class="p">),</span> <span class="n">nrow</span><span class="o">=</span><span class="m">3</span><span class="p">,</span> <span class="n">ncol</span><span class="o">=</span><span class="m">2</span><span class="p">)</span>
-<span class="n">mat_new</span> <span class="o">&lt;-</span> <span class="n">mat_1</span> <span class="o">%*%</span> <span class="n">mat_2</span>
+<span class="n">mat_1</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span><span class="w"> </span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="m">3</span><span class="p">,</span><span class="w"> </span><span class="m">4</span><span class="p">,</span><span class="w"> </span><span class="m">5</span><span class="p">,</span><span class="w"> </span><span class="m">6</span><span class="p">),</span><span class="w"> </span><span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
+<span class="n">mat_2</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span><span class="w"> </span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="m">3</span><span class="p">,</span><span class="w"> </span><span class="m">4</span><span class="p">,</span><span class="w"> </span><span class="m">5</span><span class="p">,</span><span class="w"> </span><span class="m">6</span><span class="p">),</span><span class="w"> </span><span class="n">nrow</span><span class="o">=</span><span class="m">3</span><span class="p">,</span><span class="w"> </span><span class="n">ncol</span><span class="o">=</span><span class="m">2</span><span class="p">)</span>
+<span class="n">mat_new</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="n">mat_1</span><span class="w"> </span><span class="o">%*%</span><span class="w"> </span><span class="n">mat_2</span>
 <span class="n">mat_1</span>
 <span class="n">mat_2</span>
 <span class="n">mat_new</span>
@@ -1961,9 +2101,9 @@ <h3 id="matrix">Matrix<a class="headerlink" href="#matrix" title="Permanent link
 </table>
 
 <div class="highlight"><pre><span></span><code><span class="c1"># Element-wise division</span>
-<span class="n">mat_1</span> <span class="o">&lt;-</span> <span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span> <span class="m">2</span><span class="p">,</span> <span class="m">3</span><span class="p">,</span> <span class="m">4</span><span class="p">,</span> <span class="m">5</span><span class="p">,</span> <span class="m">6</span><span class="p">),</span> <span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span> <span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
-<span class="n">mat_2</span> <span class="o">&lt;-</span> <span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span> <span class="m">2</span><span class="p">,</span> <span class="m">3</span><span class="p">,</span> <span class="m">4</span><span class="p">,</span> <span class="m">5</span><span class="p">,</span> <span class="m">6</span><span class="p">),</span> <span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span> <span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
-<span class="n">mat_new</span> <span class="o">&lt;-</span> <span class="n">mat_1</span> <span class="o">/</span> <span class="n">mat_2</span>
+<span class="n">mat_1</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span><span class="w"> </span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="m">3</span><span class="p">,</span><span class="w"> </span><span class="m">4</span><span class="p">,</span><span class="w"> </span><span class="m">5</span><span class="p">,</span><span class="w"> </span><span class="m">6</span><span class="p">),</span><span class="w"> </span><span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
+<span class="n">mat_2</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span><span class="w"> </span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="m">3</span><span class="p">,</span><span class="w"> </span><span class="m">4</span><span class="p">,</span><span class="w"> </span><span class="m">5</span><span class="p">,</span><span class="w"> </span><span class="m">6</span><span class="p">),</span><span class="w"> </span><span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
+<span class="n">mat_new</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="n">mat_1</span><span class="w"> </span><span class="o">/</span><span class="w"> </span><span class="n">mat_2</span>
 <span class="n">mat_1</span>
 <span class="n">mat_2</span>
 <span class="n">mat_new</span>
@@ -1993,8 +2133,8 @@ <h3 id="matrix">Matrix<a class="headerlink" href="#matrix" title="Permanent link
 </table>
 
 <div class="highlight"><pre><span></span><code><span class="c1"># Scalar multiplication with matrix</span>
-<span class="n">mat_1</span> <span class="o">&lt;-</span> <span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span> <span class="m">2</span><span class="p">,</span> <span class="m">3</span><span class="p">,</span> <span class="m">4</span><span class="p">,</span> <span class="m">5</span><span class="p">,</span> <span class="m">6</span><span class="p">),</span> <span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span> <span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
-<span class="n">mat_new</span> <span class="o">&lt;-</span> <span class="m">10</span> <span class="o">*</span> <span class="n">mat_1</span>
+<span class="n">mat_1</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span><span class="w"> </span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="m">3</span><span class="p">,</span><span class="w"> </span><span class="m">4</span><span class="p">,</span><span class="w"> </span><span class="m">5</span><span class="p">,</span><span class="w"> </span><span class="m">6</span><span class="p">),</span><span class="w"> </span><span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
+<span class="n">mat_new</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="m">10</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="n">mat_1</span>
 <span class="n">mat_1</span>
 <span class="n">mat_new</span>
 </code></pre></div>
@@ -2015,9 +2155,9 @@ <h3 id="matrix">Matrix<a class="headerlink" href="#matrix" title="Permanent link
 </table>
 
 <div class="highlight"><pre><span></span><code><span class="c1"># Broadcast vector of 1x3 to matrix of 2x3</span>
-<span class="n">vec_1</span> <span class="o">&lt;-</span> <span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span> <span class="m">2</span><span class="p">,</span> <span class="m">3</span><span class="p">)</span>
-<span class="n">mat_1</span> <span class="o">&lt;-</span> <span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span> <span class="m">2</span><span class="p">,</span> <span class="m">3</span><span class="p">,</span> <span class="m">4</span><span class="p">,</span> <span class="m">5</span><span class="p">,</span> <span class="m">6</span><span class="p">),</span> <span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span> <span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
-<span class="n">vec_1</span> <span class="o">+</span> <span class="n">mat_1</span>
+<span class="n">vec_1</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span><span class="w"> </span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="m">3</span><span class="p">)</span>
+<span class="n">mat_1</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span><span class="w"> </span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="m">3</span><span class="p">,</span><span class="w"> </span><span class="m">4</span><span class="p">,</span><span class="w"> </span><span class="m">5</span><span class="p">,</span><span class="w"> </span><span class="m">6</span><span class="p">),</span><span class="w"> </span><span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="n">ncol</span><span class="o">=</span><span class="m">3</span><span class="p">)</span>
+<span class="n">vec_1</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="n">mat_1</span>
 </code></pre></div>
 <table class="dataframe">
 <caption>A matrix: 2 × 3 of type dbl</caption>
@@ -2028,11 +2168,11 @@ <h3 id="matrix">Matrix<a class="headerlink" href="#matrix" title="Permanent link
 </table>
 
 <div class="highlight"><pre><span></span><code><span class="c1"># Check for determinant of matrix</span>
-<span class="n">mat_A</span> <span class="o">&lt;-</span> <span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span> <span class="m">2</span><span class="p">,</span> <span class="m">3</span><span class="p">,</span> <span class="m">4</span><span class="p">),</span> <span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span> <span class="n">ncol</span><span class="o">=</span><span class="m">2</span><span class="p">)</span>
+<span class="n">mat_A</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">matrix</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span><span class="w"> </span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="m">3</span><span class="p">,</span><span class="w"> </span><span class="m">4</span><span class="p">),</span><span class="w"> </span><span class="n">nrow</span><span class="o">=</span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="n">ncol</span><span class="o">=</span><span class="m">2</span><span class="p">)</span>
 <span class="n">mat_A</span>
 <span class="c1"># Here det != 0, hence inverse exist</span>
-<span class="n">det_of_mat_A</span> <span class="o">=</span> <span class="nf">det</span><span class="p">(</span><span class="n">mat_A</span><span class="p">)</span>
-<span class="nf">cat</span><span class="p">(</span><span class="n">det_of_mat_A</span><span class="p">,</span> <span class="s">&#39;is non-zero hence the inverse exist&#39;</span><span class="p">)</span>
+<span class="n">det_of_mat_A</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nf">det</span><span class="p">(</span><span class="n">mat_A</span><span class="p">)</span>
+<span class="nf">cat</span><span class="p">(</span><span class="n">det_of_mat_A</span><span class="p">,</span><span class="w"> </span><span class="s">&#39;is non-zero hence the inverse exist&#39;</span><span class="p">)</span>
 </code></pre></div>
 <table class="dataframe">
 <caption>A matrix: 2 × 2 of type dbl</caption>
@@ -2045,7 +2185,7 @@ <h3 id="matrix">Matrix<a class="headerlink" href="#matrix" title="Permanent link
 <div class="highlight"><pre><span></span><code>-2 is non-zero hence the inverse exist
 </code></pre></div>
 <div class="highlight"><pre><span></span><code><span class="c1"># Invert of Matrix: mat&#39;</span>
-<span class="n">mat_A_inv</span> <span class="o">=</span> <span class="nf">solve</span><span class="p">(</span><span class="n">mat_A</span><span class="p">)</span>
+<span class="n">mat_A_inv</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nf">solve</span><span class="p">(</span><span class="n">mat_A</span><span class="p">)</span>
 <span class="n">mat_A_inv</span>
 </code></pre></div>
 <table class="dataframe">
@@ -2057,10 +2197,10 @@ <h3 id="matrix">Matrix<a class="headerlink" href="#matrix" title="Permanent link
 </table>
 
 <div class="highlight"><pre><span></span><code><span class="c1"># I = A&#39;A = AA&#39;</span>
-<span class="n">mat_A_identity</span> <span class="o">=</span> <span class="n">mat_A_inv</span> <span class="o">%*%</span> <span class="n">mat_A</span>
+<span class="n">mat_A_identity</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">mat_A_inv</span><span class="w"> </span><span class="o">%*%</span><span class="w"> </span><span class="n">mat_A</span>
 <span class="n">mat_A_identity</span>
-<span class="n">mat_A_inv</span> <span class="o">%*%</span> <span class="n">mat_A</span>
-<span class="n">mat_A</span> <span class="o">%*%</span> <span class="n">mat_A_inv</span>
+<span class="n">mat_A_inv</span><span class="w"> </span><span class="o">%*%</span><span class="w"> </span><span class="n">mat_A</span>
+<span class="n">mat_A</span><span class="w"> </span><span class="o">%*%</span><span class="w"> </span><span class="n">mat_A_inv</span>
 </code></pre></div>
 <table class="dataframe">
 <caption>A matrix: 2 × 2 of type dbl</caption>
@@ -2088,8 +2228,8 @@ <h3 id="matrix">Matrix<a class="headerlink" href="#matrix" title="Permanent link
 
 <div class="highlight"><pre><span></span><code><span class="c1"># IA = AI = A</span>
 <span class="n">mat_A</span>
-<span class="n">mat_A_identity</span> <span class="o">%*%</span> <span class="n">mat_A</span>
-<span class="n">mat_A</span> <span class="o">%*%</span> <span class="n">mat_A_identity</span>
+<span class="n">mat_A_identity</span><span class="w"> </span><span class="o">%*%</span><span class="w"> </span><span class="n">mat_A</span>
+<span class="n">mat_A</span><span class="w"> </span><span class="o">%*%</span><span class="w"> </span><span class="n">mat_A_identity</span>
 </code></pre></div>
 <table class="dataframe">
 <caption>A matrix: 2 × 2 of type dbl</caption>
@@ -2116,9 +2256,9 @@ <h3 id="matrix">Matrix<a class="headerlink" href="#matrix" title="Permanent link
 </table>
 
 <h3 id="dataframes">DataFrames<a class="headerlink" href="#dataframes" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code><span class="n">price_var</span> <span class="o">&lt;-</span> <span class="nf">c</span><span class="p">(</span><span class="m">100</span><span class="p">,</span> <span class="m">105</span><span class="p">,</span> <span class="m">120</span><span class="p">)</span>
-<span class="n">date_var</span> <span class="o">&lt;-</span> <span class="nf">as.Date</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="s">&#39;2021-01-11&#39;</span><span class="p">,</span><span class="s">&#39;2021-01-12&#39;</span><span class="p">,</span><span class="s">&#39;2021-01-13&#39;</span><span class="p">))</span>
-<span class="n">df</span> <span class="o">&lt;-</span> <span class="nf">data.frame</span><span class="p">(</span><span class="n">price_var</span><span class="p">,</span> <span class="n">date_var</span><span class="p">)</span>
+<div class="highlight"><pre><span></span><code><span class="n">price_var</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">c</span><span class="p">(</span><span class="m">100</span><span class="p">,</span><span class="w"> </span><span class="m">105</span><span class="p">,</span><span class="w"> </span><span class="m">120</span><span class="p">)</span>
+<span class="n">date_var</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">as.Date</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="s">&#39;2021-01-11&#39;</span><span class="p">,</span><span class="s">&#39;2021-01-12&#39;</span><span class="p">,</span><span class="s">&#39;2021-01-13&#39;</span><span class="p">))</span>
+<span class="n">df</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">data.frame</span><span class="p">(</span><span class="n">price_var</span><span class="p">,</span><span class="w"> </span><span class="n">date_var</span><span class="p">)</span>
 <span class="n">df</span>
 </code></pre></div>
 <table class="dataframe">
@@ -2135,8 +2275,8 @@ <h3 id="dataframes">DataFrames<a class="headerlink" href="#dataframes" title="Pe
 </table>
 
 <h3 id="for-loop">For Loop<a class="headerlink" href="#for-loop" title="Permanent link">&para;</a></h3>
-<div class="highlight"><pre><span></span><code><span class="nf">for </span><span class="p">(</span><span class="n">i</span> <span class="n">in</span> <span class="nf">c</span><span class="p">(</span><span class="m">0</span><span class="o">:</span><span class="m">3</span><span class="p">))</span> <span class="p">{</span>
-    <span class="nf">print</span><span class="p">(</span><span class="n">i</span><span class="p">)</span>
+<div class="highlight"><pre><span></span><code><span class="nf">for </span><span class="p">(</span><span class="n">i</span><span class="w"> </span><span class="n">in</span><span class="w"> </span><span class="nf">c</span><span class="p">(</span><span class="m">0</span><span class="o">:</span><span class="m">3</span><span class="p">))</span><span class="w"> </span><span class="p">{</span>
+<span class="w">    </span><span class="nf">print</span><span class="p">(</span><span class="n">i</span><span class="p">)</span>
 <span class="p">}</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>[1] 0
@@ -2147,11 +2287,11 @@ <h3 id="for-loop">For Loop<a class="headerlink" href="#for-loop" title="Permanen
 <h3 id="while-loop">While Loop<a class="headerlink" href="#while-loop" title="Permanent link">&para;</a></h3>
 <div class="highlight"><pre><span></span><code><span class="c1"># Be careful, you might end up in an infinite loop</span>
 <span class="c1"># if say i = -1</span>
-<span class="n">i</span> <span class="o">&lt;-</span> <span class="m">0</span>
-<span class="nf">while </span><span class="p">(</span><span class="n">i</span> <span class="o">&lt;</span> <span class="m">3</span><span class="p">)</span> <span class="p">{</span>
-    <span class="n">stdout_str</span> <span class="o">=</span> <span class="nf">paste</span><span class="p">(</span><span class="s">&#39;Yet to reach 3, i is: &#39;</span><span class="p">,</span> <span class="n">i</span><span class="p">)</span>
-    <span class="nf">print</span><span class="p">(</span><span class="n">stdout_str</span><span class="p">)</span>
-    <span class="n">i</span> <span class="o">&lt;-</span> <span class="n">i</span> <span class="o">+</span> <span class="m">1</span>
+<span class="n">i</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="m">0</span>
+<span class="nf">while </span><span class="p">(</span><span class="n">i</span><span class="w"> </span><span class="o">&lt;</span><span class="w"> </span><span class="m">3</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
+<span class="w">    </span><span class="n">stdout_str</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nf">paste</span><span class="p">(</span><span class="s">&#39;Yet to reach 3, i is: &#39;</span><span class="p">,</span><span class="w"> </span><span class="n">i</span><span class="p">)</span>
+<span class="w">    </span><span class="nf">print</span><span class="p">(</span><span class="n">stdout_str</span><span class="p">)</span>
+<span class="w">    </span><span class="n">i</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="m">1</span>
 <span class="p">}</span>
 
 <span class="nf">print</span><span class="p">(</span><span class="s">&#39;Reached 3!&#39;</span><span class="p">)</span>
@@ -2163,19 +2303,19 @@ <h3 id="while-loop">While Loop<a class="headerlink" href="#while-loop" title="Pe
 </code></pre></div>
 <h3 id="math">Math<a class="headerlink" href="#math" title="Permanent link">&para;</a></h3>
 <div class="highlight"><pre><span></span><code><span class="c1"># Multiplication</span>
-<span class="m">2</span> <span class="o">*</span> <span class="m">2</span>
+<span class="m">2</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="m">2</span>
 </code></pre></div>
 <p>4</p>
 <div class="highlight"><pre><span></span><code><span class="c1"># Division</span>
-<span class="m">2</span> <span class="o">/</span> <span class="m">2</span>
+<span class="m">2</span><span class="w"> </span><span class="o">/</span><span class="w"> </span><span class="m">2</span>
 </code></pre></div>
 <p>1</p>
 <div class="highlight"><pre><span></span><code><span class="c1"># Addition</span>
-<span class="m">2</span> <span class="o">+</span> <span class="m">2</span>
+<span class="m">2</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="m">2</span>
 </code></pre></div>
 <p>4</p>
 <div class="highlight"><pre><span></span><code><span class="c1"># Subtraction</span>
-<span class="m">2</span> <span class="o">-</span> <span class="m">2</span>
+<span class="m">2</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="m">2</span>
 </code></pre></div>
 <p>0</p>
 <div class="highlight"><pre><span></span><code><span class="c1"># Exponentiation</span>
@@ -2191,7 +2331,7 @@ <h3 id="math">Math<a class="headerlink" href="#math" title="Permanent link">&par
 </code></pre></div>
 <p>0.693147180559945</p>
 <div class="highlight"><pre><span></span><code><span class="c1"># Log with base 2</span>
-<span class="n">var_a</span> <span class="o">=</span> <span class="nf">log</span><span class="p">(</span><span class="m">5</span><span class="p">,</span> <span class="n">base</span><span class="o">=</span><span class="m">2</span><span class="p">)</span>
+<span class="n">var_a</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nf">log</span><span class="p">(</span><span class="m">5</span><span class="p">,</span><span class="w"> </span><span class="n">base</span><span class="o">=</span><span class="m">2</span><span class="p">)</span>
 <span class="n">var_a</span>
 </code></pre></div>
 <p>2.32192809488736</p>
@@ -2230,24 +2370,24 @@ <h3 id="math">Math<a class="headerlink" href="#math" title="Permanent link">&par
 <p>NaN</p>
 <div class="highlight"><pre><span></span><code><span class="c1"># Print as fractions rather than decimal</span>
 <span class="nf">library</span><span class="p">(</span><span class="n">MASS</span><span class="p">)</span>
-<span class="nf">fractions</span><span class="p">(</span><span class="m">1</span> <span class="o">-</span> <span class="m">0.25</span><span class="p">)</span>
+<span class="nf">fractions</span><span class="p">(</span><span class="m">1</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="m">0.25</span><span class="p">)</span>
 </code></pre></div>
 <p>&frac34;</p>
 <h3 id="statistics">Statistics<a class="headerlink" href="#statistics" title="Permanent link">&para;</a></h3>
 <div class="highlight"><pre><span></span><code><span class="c1"># Create list</span>
-<span class="n">list_a</span> <span class="o">&lt;-</span> <span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span> <span class="m">2</span><span class="p">,</span> <span class="m">3</span><span class="p">,</span> <span class="m">4</span><span class="p">,</span> <span class="m">5</span><span class="p">,</span> <span class="m">6</span><span class="p">)</span>
+<span class="n">list_a</span><span class="w"> </span><span class="o">&lt;-</span><span class="w"> </span><span class="nf">c</span><span class="p">(</span><span class="m">1</span><span class="p">,</span><span class="w"> </span><span class="m">2</span><span class="p">,</span><span class="w"> </span><span class="m">3</span><span class="p">,</span><span class="w"> </span><span class="m">4</span><span class="p">,</span><span class="w"> </span><span class="m">5</span><span class="p">,</span><span class="w"> </span><span class="m">6</span><span class="p">)</span>
 
 <span class="c1"># Get standard deviation</span>
-<span class="nf">cat</span><span class="p">(</span><span class="s">&#39;standard deviation:&#39;</span><span class="p">,</span> <span class="nf">sd</span><span class="p">(</span><span class="n">list_a</span><span class="p">))</span>
+<span class="nf">cat</span><span class="p">(</span><span class="s">&#39;standard deviation:&#39;</span><span class="p">,</span><span class="w"> </span><span class="nf">sd</span><span class="p">(</span><span class="n">list_a</span><span class="p">))</span>
 
 <span class="c1"># Get variance</span>
-<span class="nf">cat</span><span class="p">(</span><span class="s">&#39;\nvariance:&#39;</span><span class="p">,</span> <span class="nf">var</span><span class="p">(</span><span class="n">list_a</span><span class="p">))</span>
+<span class="nf">cat</span><span class="p">(</span><span class="s">&#39;\nvariance:&#39;</span><span class="p">,</span><span class="w"> </span><span class="nf">var</span><span class="p">(</span><span class="n">list_a</span><span class="p">))</span>
 
 <span class="c1"># Get mean</span>
-<span class="nf">cat</span><span class="p">(</span><span class="s">&#39;\nmedian:&#39;</span><span class="p">,</span> <span class="nf">mean</span><span class="p">(</span><span class="n">list_a</span><span class="p">))</span>
+<span class="nf">cat</span><span class="p">(</span><span class="s">&#39;\nmedian:&#39;</span><span class="p">,</span><span class="w"> </span><span class="nf">mean</span><span class="p">(</span><span class="n">list_a</span><span class="p">))</span>
 
 <span class="c1"># Get median</span>
-<span class="nf">cat</span><span class="p">(</span><span class="s">&#39;\nmean:&#39;</span><span class="p">,</span> <span class="nf">median</span><span class="p">(</span><span class="n">list_a</span><span class="p">))</span>
+<span class="nf">cat</span><span class="p">(</span><span class="s">&#39;\nmean:&#39;</span><span class="p">,</span><span class="w"> </span><span class="nf">median</span><span class="p">(</span><span class="n">list_a</span><span class="p">))</span>
 
 <span class="c1"># Get summary statistics</span>
 <span class="nf">summary</span><span class="p">(</span><span class="n">list_a</span><span class="p">)</span>
@@ -2262,7 +2402,7 @@ <h3 id="statistics">Statistics<a class="headerlink" href="#statistics" title="Pe
    1.00    2.25    3.50    3.50    4.75    6.00
 </code></pre></div>
 <div class="highlight"><pre><span></span><code><span class="c1"># t-test</span>
-<span class="n">var_test</span> <span class="o">=</span> <span class="nf">t.test</span><span class="p">(</span><span class="n">list_a</span><span class="p">)</span>
+<span class="n">var_test</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nf">t.test</span><span class="p">(</span><span class="n">list_a</span><span class="p">)</span>
 <span class="n">var_test</span>
 </code></pre></div>
 <div class="highlight"><pre><span></span><code>    One Sample t-test
@@ -2315,7 +2455,7 @@ <h3 id="statistics">Statistics<a class="headerlink" href="#statistics" title="Pe
 <div class="highlight"><pre><span></span><code><span class="c1"># Plot dot plot</span>
 <span class="c1"># type: p for point, l for line, b for both</span>
 <span class="c1"># col: cyan, blue, green, red</span>
-<span class="nf">plot</span><span class="p">(</span><span class="n">list_a</span><span class="p">,</span> <span class="n">xlab</span><span class="o">=</span><span class="s">&#39;variable x&#39;</span><span class="p">,</span> <span class="n">ylab</span><span class="o">=</span><span class="s">&#39;variable y&#39;</span><span class="p">,</span> <span class="n">main</span><span class="o">=</span><span class="s">&#39;X-Y Plot&#39;</span><span class="p">,</span> <span class="n">type</span><span class="o">=</span><span class="s">&#39;b&#39;</span><span class="p">,</span> <span class="n">col</span><span class="o">=</span><span class="s">&#39;blue&#39;</span><span class="p">)</span>
+<span class="nf">plot</span><span class="p">(</span><span class="n">list_a</span><span class="p">,</span><span class="w"> </span><span class="n">xlab</span><span class="o">=</span><span class="s">&#39;variable x&#39;</span><span class="p">,</span><span class="w"> </span><span class="n">ylab</span><span class="o">=</span><span class="s">&#39;variable y&#39;</span><span class="p">,</span><span class="w"> </span><span class="n">main</span><span class="o">=</span><span class="s">&#39;X-Y Plot&#39;</span><span class="p">,</span><span class="w"> </span><span class="n">type</span><span class="o">=</span><span class="s">&#39;b&#39;</span><span class="p">,</span><span class="w"> </span><span class="n">col</span><span class="o">=</span><span class="s">&#39;blue&#39;</span><span class="p">)</span>
 </code></pre></div>
 <p><img alt="png" src="../r_files/r_62_0.png" /></p>
 <div class="highlight"><pre><span></span><code>
@@ -2389,7 +2529,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -2452,12 +2592,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/programming/sympy/calculus_sympy/index.html b/programming/sympy/calculus_sympy/index.html
index a45164a35..88780237b 100644
--- a/programming/sympy/calculus_sympy/index.html
+++ b/programming/sympy/calculus_sympy/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/programming/sympy/calculus_sympy/">
       
       <link rel="icon" href="../../../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -261,6 +261,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../../../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -916,11 +932,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -948,14 +1088,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1010,14 +1150,14 @@
     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" checked>
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" checked>
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1406,14 +1546,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1429,14 +1569,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1489,14 +1629,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -2218,7 +2358,7 @@ <h2 id="__comments">Comments</h2>
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -2281,12 +2421,16 @@ <h2 id="__comments">Comments</h2>
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../../../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../../../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../../../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/review/index.html b/review/index.html
index a17191af2..b2e4fe00c 100644
--- a/review/index.html
+++ b/review/index.html
@@ -16,7 +16,7 @@
         <link rel="canonical" href="https://www.deeplearningwizard.com/review/">
       
       <link rel="icon" href="../docs/assets/favicon.ico">
-      <meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.2.4">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-7.2.4">
     
     
       
@@ -186,7 +186,7 @@
     
       <div class="md-header__source">
         
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -263,6 +263,22 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../language_model/intro/" class="md-tabs__link">
+        Language Model
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+
+
+  
+  
+  
     <li class="md-tabs__item">
       <a href="../machine_learning/intro/" class="md-tabs__link">
         Machine Learning
@@ -357,7 +373,7 @@
   
     <div class="md-nav__source">
       
-<a href="https://github.com/ritchieng/deep-learning-wizard/" title="Go to repository" class="md-source" data-md-component="source">
+<a href="https://github.com/ritchieng/deep-learning-wizard" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
@@ -928,11 +944,135 @@
         <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" >
       
       <label class="md-nav__link" for="__nav_4">
+        Language Model
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Language Model" data-md-level="1">
+        <label class="md-nav__title" for="__nav_4">
+          <span class="md-nav__icon md-icon"></span>
+          Language Model
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../language_model/intro/" class="md-nav__link">
+        Introduction
+      </a>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+      
+      <label class="md-nav__link" for="__nav_4_2">
+        Containers
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="Containers" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_2">
+          <span class="md-nav__icon md-icon"></span>
+          Containers
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../language_model/containers/hpc_containers_apptainer/" class="md-nav__link">
+        HPC Containers with Apptainer
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+            
+  
+  
+  
+    
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_3" type="checkbox" id="__nav_4_3" >
+      
+      <label class="md-nav__link" for="__nav_4_3">
+        LLMs
+        <span class="md-nav__icon md-icon"></span>
+      </label>
+      <nav class="md-nav" aria-label="LLMs" data-md-level="2">
+        <label class="md-nav__title" for="__nav_4_3">
+          <span class="md-nav__icon md-icon"></span>
+          LLMs
+        </label>
+        <ul class="md-nav__list" data-md-scrollfix>
+          
+            
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../language_model/llm/llm_intro_hyperparameter_tuning/" class="md-nav__link">
+        LLMs Introduction and Hyperparameter Tuning
+      </a>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+          
+        </ul>
+      </nav>
+    </li>
+  
+
+    
+      
+      
+      
+
+  
+  
+  
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+      
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+      
+      <label class="md-nav__link" for="__nav_5">
         Machine Learning
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Machine Learning" data-md-level="1">
-        <label class="md-nav__title" for="__nav_4">
+        <label class="md-nav__title" for="__nav_5">
           <span class="md-nav__icon md-icon"></span>
           Machine Learning
         </label>
@@ -960,14 +1100,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4_2" type="checkbox" id="__nav_4_2" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_2" type="checkbox" id="__nav_5_2" >
       
-      <label class="md-nav__link" for="__nav_4_2">
+      <label class="md-nav__link" for="__nav_5_2">
         RAPIDS cuDF
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="RAPIDS cuDF" data-md-level="2">
-        <label class="md-nav__title" for="__nav_4_2">
+        <label class="md-nav__title" for="__nav_5_2">
           <span class="md-nav__icon md-icon"></span>
           RAPIDS cuDF
         </label>
@@ -1020,14 +1160,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
       
-      <label class="md-nav__link" for="__nav_5">
+      <label class="md-nav__link" for="__nav_6">
         Programming
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Programming" data-md-level="1">
-        <label class="md-nav__title" for="__nav_5">
+        <label class="md-nav__title" for="__nav_6">
           <span class="md-nav__icon md-icon"></span>
           Programming
         </label>
@@ -1158,14 +1298,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6" type="checkbox" id="__nav_6" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
       
-      <label class="md-nav__link" for="__nav_6">
+      <label class="md-nav__link" for="__nav_7">
         Data Engineering
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Data Engineering" data-md-level="1">
-        <label class="md-nav__title" for="__nav_6">
+        <label class="md-nav__title" for="__nav_7">
           <span class="md-nav__icon md-icon"></span>
           Data Engineering
         </label>
@@ -1181,14 +1321,14 @@
     <li class="md-nav__item md-nav__item--section md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_6_1" type="checkbox" id="__nav_6_1" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7_1" type="checkbox" id="__nav_7_1" >
       
-      <label class="md-nav__link" for="__nav_6_1">
+      <label class="md-nav__link" for="__nav_7_1">
         Cassandra (NoSQL)
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="Cassandra (NoSQL)" data-md-level="2">
-        <label class="md-nav__title" for="__nav_6_1">
+        <label class="md-nav__title" for="__nav_7_1">
           <span class="md-nav__icon md-icon"></span>
           Cassandra (NoSQL)
         </label>
@@ -1241,14 +1381,14 @@
     <li class="md-nav__item md-nav__item--nested">
       
       
-        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_7" type="checkbox" id="__nav_7" >
+        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_8" type="checkbox" id="__nav_8" >
       
-      <label class="md-nav__link" for="__nav_7">
+      <label class="md-nav__link" for="__nav_8">
         News
         <span class="md-nav__icon md-icon"></span>
       </label>
       <nav class="md-nav" aria-label="News" data-md-level="1">
-        <label class="md-nav__title" for="__nav_7">
+        <label class="md-nav__title" for="__nav_8">
           <span class="md-nav__icon md-icon"></span>
           News
         </label>
@@ -1535,7 +1675,7 @@ <h1 id="reviews">Reviews<a class="headerlink" href="#reviews" title="Permanent l
       <div class="md-footer-copyright">
         
           <div class="md-footer-copyright__highlight">
-            Copyright &copy; 2018-2023 Deep Learning Wizard by Ritchie Ng
+            Copyright &copy; 2018-2024 Deep Learning Wizard by Ritchie Ng
           </div>
         
         
@@ -1598,12 +1738,16 @@ <h1 id="reviews">Reviews<a class="headerlink" href="#reviews" title="Permanent l
     <div class="md-dialog" data-md-component="dialog">
       <div class="md-dialog__inner md-typeset"></div>
     </div>
-    <script id="__config" type="application/json">{"base": "..", "features": ["content.code.annotate", "content.tabs.link", "navigation.sections", "navigation.tabs", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../assets/javascripts/workers/search.709b4209.min.js", "version": null}</script>
+    <script id="__config" type="application/json">{"base": "..", "features": ["content.code.annotate", "content.tabs.link", "content.code.copy", "content.action.edit", "content.action.view", "navigation.sections", "navigation.tabs", "navigation.footer", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest"], "search": "../assets/javascripts/workers/search.709b4209.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": null}</script>
     
     
       <script src="../assets/javascripts/bundle.56838a2c.min.js"></script>
       
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML"></script>
+        <script src="../javascripts/mathjax.js"></script>
+      
+        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+      
+        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
       
     
   </body>
diff --git a/search/search_index.json b/search/search_index.json
index 7268278b5..9302fc191 100644
--- a/search/search_index.json
+++ b/search/search_index.json
@@ -1 +1 @@
-{"config":{"indexing":"full","lang":["en"],"min_search_length":3,"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"","title":"Home"},{"location":"about/","text":"About Us \u00b6 We deploy a top-down approach that enables you to grasp deep learning and deep reinforcement learning theories and code easily and quickly. We have open-sourced all our materials through our Deep Learning Wizard Tutorials . For visual learners, feel free to sign up for our video course and join over 6000 deep learning wizards. To this date, we have taught thousands of students across more than 120+ countries from students in high school to postgraduates and professionals in leading MNCs and research institutions around the world. Experienced Research and Applied Core Team \u00b6 Ritchie Ng Industry : As the Group Head of Data & AI at Eastspring Investments, I spearhead engineering, governance, and strategy across data and machine learning in 13 countries with Assets Under Management (AUM) exceeding USD 220 billion. Academia : Serving in the capacity of a Visiting Research Scholar at NExT++ Lab, jointly setup between NUS and Tsinghua University, I focus on applied deep learning research for Asian systematic trading strategies and the development of financial Large Language Models (LLM) using multimodal and multilingual data. Education : Master's degree from Imperial College London and Bachelor's degree from NUS. About : ritchieng.live Notes : ritchieng.com Guides : deeplearningwizard.com Jie Fu Jie Fu (\u4ed8\u6770) is a happy and funny machine learning researcher (principal investigator), at Beijing Academy of Artificial Intelligence2 (BAAI, \u5317\u4eac\u667a\u6e90\u4eba\u5de5\u667a\u80fd\u7814\u7a76\u9662), with his human-friendly big AI dream. He worked as a postdoc with Yoshua Bengio at University of Montreal, Quebec AI Institute (Mila), funded by Microsoft Research Montreal. He was an IVADO postdoc fellow working with Chris Pal at Polytechnique Montreal, Quebec AI Institute (Mila). He obtained his PhD from National University of Singapore under the supervision of Tat-Seng Chua. He received ICLR 2021 Outstanding Paper Award. He is currently working towards system-2 deep learning and its adaption to various real-world tasks, including AI for Science (drug discovery in particular) and AI for Social Good that can benefit all of society. He is also broadly interested in general deep learning, reinforcement learning, and language processing. Check out my profile link at bigaidream.github.io Supporters \u00b6 Alfredo Canziani Alfredo is the main person who inspired Ritchie to open-source Deep Learning Wizard's materials. We are very grateful for his advice particular in the space of open-source projects, deep learning and PyTorch. He is currently working as an Assistant Professor at Courant Institute of Mathematical Sciences, under the guidance of professor Yann LeCun. Do check out his latest mini-course on PyTorch that was held in Princeton University and was delivered with Ritchie Ng at AMMI (AIMS) Kigali, Rwanda supported by Google and Facebook. Marek Bardonski Since graduation, Marek has been working on state-of-the-art research. He was involved in inventing the system that diagnoses problems with Ubuntu packages using Machine Learning and in improving the detection accuracy of breast cancer using Deep Learning, each time leading 40 highly skilled Machine Learning/Deep Learning engineers for over 6 months. NASA noticed his unique skills and asked him to manage a project aiming to improve the International Space Station rotation algorithm that would allow them to collect enough power from solar panels while preventing them from overheating. The project was successful and generated a huge ROI for NASA. Marek has been honored by multiple Harvard professors and the Director of Advanced Exploration Systems at NASA, who signed an official Letter of Recommendation. Since his first summer of university coursework, Marek has participated in four internships at NVIDIA HQ and Microsoft HQ. He also spent one year in NVIDIA Switzerland optimizing Deep Learning inference for self-driving cars and, by implementing very efficient Deep Learning algorithms in the GPU assembly language, improving the speed of Tesla\u2019s cars self-driving engine inference. He's currently the Head of AI at Sigmoidal . \"NVIDIA Inception Partner Deep Learning Wizard is proudly an NVIDIA Inception Partner, empowering people across the world to leverage on deep learning via open-source programming languages and frameworks to solve problems. Amazon AWS Activate Programme Startup Deep Learning Wizard is also supported by the Amazon AWS Activate Programme, Portfolio Plus, allowing us to have the infrastructure to scale and grow. Citation \u00b6 If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"About Us"},{"location":"about/#about-us","text":"We deploy a top-down approach that enables you to grasp deep learning and deep reinforcement learning theories and code easily and quickly. We have open-sourced all our materials through our Deep Learning Wizard Tutorials . For visual learners, feel free to sign up for our video course and join over 6000 deep learning wizards. To this date, we have taught thousands of students across more than 120+ countries from students in high school to postgraduates and professionals in leading MNCs and research institutions around the world.","title":"About Us"},{"location":"about/#experienced-research-and-applied-core-team","text":"Ritchie Ng Industry : As the Group Head of Data & AI at Eastspring Investments, I spearhead engineering, governance, and strategy across data and machine learning in 13 countries with Assets Under Management (AUM) exceeding USD 220 billion. Academia : Serving in the capacity of a Visiting Research Scholar at NExT++ Lab, jointly setup between NUS and Tsinghua University, I focus on applied deep learning research for Asian systematic trading strategies and the development of financial Large Language Models (LLM) using multimodal and multilingual data. Education : Master's degree from Imperial College London and Bachelor's degree from NUS. About : ritchieng.live Notes : ritchieng.com Guides : deeplearningwizard.com Jie Fu Jie Fu (\u4ed8\u6770) is a happy and funny machine learning researcher (principal investigator), at Beijing Academy of Artificial Intelligence2 (BAAI, \u5317\u4eac\u667a\u6e90\u4eba\u5de5\u667a\u80fd\u7814\u7a76\u9662), with his human-friendly big AI dream. He worked as a postdoc with Yoshua Bengio at University of Montreal, Quebec AI Institute (Mila), funded by Microsoft Research Montreal. He was an IVADO postdoc fellow working with Chris Pal at Polytechnique Montreal, Quebec AI Institute (Mila). He obtained his PhD from National University of Singapore under the supervision of Tat-Seng Chua. He received ICLR 2021 Outstanding Paper Award. He is currently working towards system-2 deep learning and its adaption to various real-world tasks, including AI for Science (drug discovery in particular) and AI for Social Good that can benefit all of society. He is also broadly interested in general deep learning, reinforcement learning, and language processing. Check out my profile link at bigaidream.github.io","title":"Experienced Research and Applied Core Team"},{"location":"about/#supporters","text":"Alfredo Canziani Alfredo is the main person who inspired Ritchie to open-source Deep Learning Wizard's materials. We are very grateful for his advice particular in the space of open-source projects, deep learning and PyTorch. He is currently working as an Assistant Professor at Courant Institute of Mathematical Sciences, under the guidance of professor Yann LeCun. Do check out his latest mini-course on PyTorch that was held in Princeton University and was delivered with Ritchie Ng at AMMI (AIMS) Kigali, Rwanda supported by Google and Facebook. Marek Bardonski Since graduation, Marek has been working on state-of-the-art research. He was involved in inventing the system that diagnoses problems with Ubuntu packages using Machine Learning and in improving the detection accuracy of breast cancer using Deep Learning, each time leading 40 highly skilled Machine Learning/Deep Learning engineers for over 6 months. NASA noticed his unique skills and asked him to manage a project aiming to improve the International Space Station rotation algorithm that would allow them to collect enough power from solar panels while preventing them from overheating. The project was successful and generated a huge ROI for NASA. Marek has been honored by multiple Harvard professors and the Director of Advanced Exploration Systems at NASA, who signed an official Letter of Recommendation. Since his first summer of university coursework, Marek has participated in four internships at NVIDIA HQ and Microsoft HQ. He also spent one year in NVIDIA Switzerland optimizing Deep Learning inference for self-driving cars and, by implementing very efficient Deep Learning algorithms in the GPU assembly language, improving the speed of Tesla\u2019s cars self-driving engine inference. He's currently the Head of AI at Sigmoidal . \"NVIDIA Inception Partner Deep Learning Wizard is proudly an NVIDIA Inception Partner, empowering people across the world to leverage on deep learning via open-source programming languages and frameworks to solve problems. Amazon AWS Activate Programme Startup Deep Learning Wizard is also supported by the Amazon AWS Activate Programme, Portfolio Plus, allowing us to have the infrastructure to scale and grow.","title":"Supporters"},{"location":"about/#citation","text":"If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Citation"},{"location":"consultancy/","text":"Training, Consultancy & Deployment \u00b6 We've empowered hundreds of clients from large start-ups, to MNCs, to educational institutions, and to government organizations with deep learning. If you require on-site training, consultancy or deployment on deep learning, deep reinforcement learning, machine learning, and/or general data science projects, please feel free to contact us at ritchie@deeplearningwizard.com .","title":"Consultancy"},{"location":"consultancy/#training-consultancy-deployment","text":"We've empowered hundreds of clients from large start-ups, to MNCs, to educational institutions, and to government organizations with deep learning. If you require on-site training, consultancy or deployment on deep learning, deep reinforcement learning, machine learning, and/or general data science projects, please feel free to contact us at ritchie@deeplearningwizard.com .","title":"Training, Consultancy &amp; Deployment"},{"location":"home/","text":"Learn The Whole AI Pipeline With \u00b6 You will the learn the suite of tools to build an end-to-end deep learning pipeline. Tools You Will Learn To Use PyTorch (CPU/GPU) Scikit-learn (CPU) RAPIDS cuML (GPU) Gym (CPU) Python (scripting) C++ (programming) Bash (scripting) NumPy (CPU) CuPy (GPU) Pandas (CPU) RAPIDS cuDF (GPU) Matplotlib (Plot) Plotly (Plot) Streamlit (Dashboard) CassandraDB (CPU DB) BlazingSQL (GPU DB) We deploy a top-down approach that enables you to grasp deep learning and deep reinforcement learning theories and code easily and quickly. We have open-sourced all our materials through our Deep Learning Wizard Tutorials . For visual learners, feel free to sign up for our video course and join over 6000 deep learning wizards. To this date, we have taught thousands of students across more than 120+ countries from students in high school to postgraduates and professionals in leading MNCs and research institutions around the world. Simulation of deep reinforcement learning agent mastering games like Super Mario Bros , Flappy Bird and PacMan . These games have APIs for algorithms to interact with the environment, and they are created by talented people so feel free to check out their respective repositories with the links given. \u21a9","title":"Intro"},{"location":"home/#learn-the-whole-ai-pipeline-with","text":"You will the learn the suite of tools to build an end-to-end deep learning pipeline. Tools You Will Learn To Use PyTorch (CPU/GPU) Scikit-learn (CPU) RAPIDS cuML (GPU) Gym (CPU) Python (scripting) C++ (programming) Bash (scripting) NumPy (CPU) CuPy (GPU) Pandas (CPU) RAPIDS cuDF (GPU) Matplotlib (Plot) Plotly (Plot) Streamlit (Dashboard) CassandraDB (CPU DB) BlazingSQL (GPU DB) We deploy a top-down approach that enables you to grasp deep learning and deep reinforcement learning theories and code easily and quickly. We have open-sourced all our materials through our Deep Learning Wizard Tutorials . For visual learners, feel free to sign up for our video course and join over 6000 deep learning wizards. To this date, we have taught thousands of students across more than 120+ countries from students in high school to postgraduates and professionals in leading MNCs and research institutions around the world. Simulation of deep reinforcement learning agent mastering games like Super Mario Bros , Flappy Bird and PacMan . These games have APIs for algorithms to interact with the environment, and they are created by talented people so feel free to check out their respective repositories with the links given. \u21a9","title":"Learn The Whole AI Pipeline With"},{"location":"pipeline/","text":"CPU to GPU Production-level Pipeline for AI \u00b6 At Deep Learning Wizard, we cover the basics of some parts of the whole tech stack for production-level CPU/GPU-powered AI. This AI pipeline is entirely based on open-source distributions. This stack would get you started, and enable you to adjust the stack according to your needs.","title":"CPU to GPU Production-level Pipeline for AI"},{"location":"pipeline/#cpu-to-gpu-production-level-pipeline-for-ai","text":"At Deep Learning Wizard, we cover the basics of some parts of the whole tech stack for production-level CPU/GPU-powered AI. This AI pipeline is entirely based on open-source distributions. This stack would get you started, and enable you to adjust the stack according to your needs.","title":"CPU to GPU Production-level Pipeline for AI"},{"location":"review/","text":"Reviews \u00b6 To this date, we have taught thousands of students across more than 120+ countries from students as young as 15 to postgraduates and professionals in leading MNCs and research institutions around the world. These are just some of the hundreds of reviews we've had and will be updated when we have more time or contributors. Roberto Trevi\u00f1o Cervantes Congratulations! You have achieved the unachievable, teaching a 15 year old boy how to get a machine to estimate a function, just what he had been trying to do for almost a year. Muktabh Mayank This course helped me understand idiomatic pytorch and avoiding translating theano-to-torch. Charles Neiswender I really feel like I can take the techniques I learned and apply them to deep learning projects of my choosing. Ian Lipton This was a good overview of the different types of neural networks. I like that the instructor started with linear and logistic regression before diving into the neural networks. That background helped make sense of the neural network functionality. I like that this gave enough detail on the mathematics so that you can understand the basic functioning of the models but didn't overwhelm you with the nitty gritty math. And check out hundreds of more reviews for our video course !","title":"Reviews"},{"location":"review/#reviews","text":"To this date, we have taught thousands of students across more than 120+ countries from students as young as 15 to postgraduates and professionals in leading MNCs and research institutions around the world. These are just some of the hundreds of reviews we've had and will be updated when we have more time or contributors. Roberto Trevi\u00f1o Cervantes Congratulations! You have achieved the unachievable, teaching a 15 year old boy how to get a machine to estimate a function, just what he had been trying to do for almost a year. Muktabh Mayank This course helped me understand idiomatic pytorch and avoiding translating theano-to-torch. Charles Neiswender I really feel like I can take the techniques I learned and apply them to deep learning projects of my choosing. Ian Lipton This was a good overview of the different types of neural networks. I like that the instructor started with linear and logistic regression before diving into the neural networks. That background helped make sense of the neural network functionality. I like that this gave enough detail on the mathematics so that you can understand the basic functioning of the models but didn't overwhelm you with the nitty gritty math. And check out hundreds of more reviews for our video course !","title":"Reviews"},{"location":"data_engineering/nosql/cassandra/intro/","text":"Apache Cassandra \u00b6 Tools You Will Learn To Use Apache Cassandra Reasons for Using Apache Cassandra \u00b6 It is is an open-source NoSQL database management system (DBMS) that is designed to handle large amounts of data across many servers, providing high availability with no single point of failure. Meaning if one server fails, everything goes as per normal as data is shared amongst nodes (servers). Compared to other master/slave models like MongoDB, Cassandra has a master-less model. It has extremely fast read/write speeds. You read/write millions of rows of data easily. Work in progress This open-source portion is still a work in progress. Stay tuned while we gradually upload our tutorials and notes. Feel free to contact Ritchie Ng if you would like to contribute via our Facebook page.","title":"Introduction"},{"location":"data_engineering/nosql/cassandra/intro/#apache-cassandra","text":"Tools You Will Learn To Use Apache Cassandra","title":"Apache Cassandra"},{"location":"data_engineering/nosql/cassandra/intro/#reasons-for-using-apache-cassandra","text":"It is is an open-source NoSQL database management system (DBMS) that is designed to handle large amounts of data across many servers, providing high availability with no single point of failure. Meaning if one server fails, everything goes as per normal as data is shared amongst nodes (servers). Compared to other master/slave models like MongoDB, Cassandra has a master-less model. It has extremely fast read/write speeds. You read/write millions of rows of data easily. Work in progress This open-source portion is still a work in progress. Stay tuned while we gradually upload our tutorials and notes. Feel free to contact Ritchie Ng if you would like to contribute via our Facebook page.","title":"Reasons for Using Apache Cassandra"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/","text":"Setting up Cassandra Multi-node Cluster \u00b6 Install Apache Cassandra \u00b6 Patience is key initially It will be quite tedious to many people in setting up a multi-node Cassandra cluster. You are likely to face a lot of problems. But with a strong community and this tried-and-proven guide, you should be ok! Just be patient. Remember when you're asking for help on StackOverflow, here or anywhere else, do paste the logs to make everyone's life easier to help you debug quickly. All you need to do is to run this base command and paste the logs. cat /var/log/cassandra/system.log Here, we will install all of the required Debian packages. To ensure you've the latest version of Cassandra, please use the official link at cassandra.apache.org/download . Bash Commands echo \"deb http://www.apache.org/dist/cassandra/debian 311x main\" | sudo tee -a /etc/apt/sources.list.d/cassandra.sources.list curl https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add - sudo apt-get update sudo apt-get install cassandra This would get you up and running with Cassandra v3.11. Install DataStax Python Cassandra Driver \u00b6 We need to install the Python Cassandra Driver so we can easily interact with our Cassandra Database without using the CQLSH (this is Cassandra's shell to run CQL commands to perform CRUD operations) commandline directly. What is CRUD? CRUD is the acronym of CREATE, READ, UPDATE, and DELETE. For databases, we typically have these 4 categories of operations so we can create new data points, update, read or delete them in our database. Bash Commands pip install cassandra-driver This is strictly for installation on Linux platforms, refer to the official website for more details. Single Node Cassandra Cluster \u00b6 Before we venture into the cool world of multi-node cluster requiring many servers, we will start with a single node cluster that only requires a single server (desktop/laptop). Once you've installed everything so far, you should run the following. Bash commands sudo service cassandra start sudo nodetool status And this will print out something like that: Datacenter: datacenter1 ============================= Status=Up/Down |/ State=Normal/Leaving/Joining/Moving -- Address Load Tokens Owns (effective) Host ID Rack UN 127.0.0.1 318.78 KiB 256 100.0% g5ac4c9-99b7-65d-24cfd82524f9 rack1 That's it, we've built our single node Cassandra database. Multi-node Cassandra Cluster \u00b6 Remember it's standard to have at least 3 nodes, and in a basic 3 separate server configuration (3 separate desktops for non-enterprise users). Because of this, we will base this tutorial on setting up a 3-node cluster. But the same steps apply if you want to even do 10000 nodes. Installing Sublime critical Sublime is a text editor, and it would help if you're not familiar with VIM which I use frequently. Honestly using Sublime to edit all the following files we will edit is much easier, trust me! To install Sublime run the following bash commands in sequence: wget -qO - https://download.sublimetext.com/sublimehq-pub.gpg | sudo apt-key add - echo \"deb https://download.sublimetext.com/ apt/stable/\" | sudo tee /etc/apt/sources.list.d/sublime-text.list sudo apt-get update sudo apt-get install sublime-text Now check if it works by running this command in your bash: subl This should open Sublime text editor! You're now ready. Before we move on to the steps to set up each node, we need the IP address of all 3 servers. This is simple, just run the following bash command: ifconfig | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\\.){3}[0-9]*).*/\\2/p' This would return the server's IP address that we need for example: server_1_ip: 112.522.6.61 server_2_ip: 112.522.6.62 server_3_ip: 112.522.6.63 We will be using these IPs as a base for configuration in the subsequent sections. Take note yours would differ and you need to change accordingly. Steps Per Node \u00b6 Critical Section You Need To Repeat This is a critcal section. On EVERY server, you need to repeat all the steps shown in this section. In our case of a 3 node cluster, using 3 servers, we need to repeat this 3 times. Step 1: Modify Cassandra Configuration Settings \u00b6 Run the following bash to edit the configuration file for server 1 (112.522.6.61): cd /etc/cassandra subl cassandra.yaml Now you need to find the following fields and change them accordingly. cluster_name: 'CassandraDBCluster' seed_provider: - class_name: org.apache.cassandra.locator.SimpleSeedProvider parameters: - seeds: \"112.522.6.61, 112.522.6.62, 112.522.6.63\" listen_address: 112.522.6.61 rpc_address: 112.522.6.61 endpoint_snitch: GossipingPropertyFileSnitch auto_bootstrap: true Step 2: Modify Rack Details \u00b6 Run the following bash command to edit the rack details: cd /etc/cassandra subl cassandra-rackdc.properties And change the following fields to this: dc=datacenter1 Step 3: Check Status \u00b6 Restart service with the following bash commands: sudo rm -rf /var/lib/cassandra/data/system/* sudo service cassandra restart sudo nodetool status Here you would see an output like this: Datacenter: datacenter ============================= Status=Up/Down |/ State=Normal/Leaving/Joining/Moving -- Address Load Tokens Owns (effective) Host ID Rack UN 112.522.6.61 318.78 KiB 256 100.0% f5c84c9-99b7-45d-8856-24cfd82523f9 rack1 UN 112.522.6.62 206.42 KiB 256 100.0% ac2f24da-1b2c-4e3-8a75-0e28ec366a4 rack1 UN 112.522.6.63 338.34 KiB 256 100.0% 55a227d-ffbe-45d5-8698-60c374b3a6b rack1 Step 4: Configure Firewall IP Settings \u00b6 Run the following bash commands sudo apt-get install iptables-persistent sudo iptables -A INPUT -p tcp -s 112.522.6.62 -m multiport --dports 7000,9042 -m state --state NEW,ESTABLISHED -j ACCEPT sudo iptables -A INPUT -p tcp -s 112.522.6.63 -m multiport --dports 7000,9042 -m state --state NEW,ESTABLISHED -j ACCEPT sudo bash -c \"iptables-save > /etc/iptables/rules.v4\" sudo netfilter-persistent reload sudo nodetool status Repeat Steps 1 to 4 \u00b6 You have to repeat all 4 steps for the other 2 servers but step 1 and step 4 requires slightly different settings. For the other servers, you need to basically change the IP to the server's IP. And for the IP firewall settings, you need to allow entry from the other 2 servers instead of itself. Server 2 Example for Step 2 \u00b6 cluster_name: 'CassandraDBCluster' seed_provider: - class_name: org.apache.cassandra.locator.SimpleSeedProvider parameters: - seeds: \"112.522.6.61, 112.522.6.62, 112.522.6.63\" # ONLY THIS LINE CHANGES listen_address: 112.522.6.62 # ONLY THIS LINE CHANGES rpc_address: 112.522.6.62 endpoint_snitch: GossipingPropertyFileSnitch auto_bootstrap: true Server 3 Example for Step 2 \u00b6 cluster_name: 'CassandraDBCluster' seed_provider: - class_name: org.apache.cassandra.locator.SimpleSeedProvider parameters: - seeds: \"112.522.6.61, 112.522.6.62, 112.522.6.63\" # ONLY THIS LINE CHANGES listen_address: 112.522.6.63 # ONLY THIS LINE CHANGES rpc_address: 112.522.6.63 endpoint_snitch: GossipingPropertyFileSnitch auto_bootstrap: true Server 2 Example for Step 4 \u00b6 sudo apt-get install iptables-persistent # ONLY THESE LINES CHANGE TO ALLOW COMMUNICATION WITH SERVER 1 AND 3 sudo iptables -A INPUT -p tcp -s 112.522.6.61 -m multiport --dports 7000,9042 -m state --state NEW,ESTABLISHED -j ACCEPT sudo iptables -A INPUT -p tcp -s 112.522.6.63 -m multiport --dports 7000,9042 -m state --state NEW,ESTABLISHED -j ACCEPT sudo bash -c \"iptables-save > /etc/iptables/rules.v4\" sudo netfilter-persistent reload sudo nodetool status Server 3 Example for Step 4 \u00b6 sudo apt-get install iptables-persistent # ONLY THESE LINES CHANGE TO ALLOW COMMUNICATION WITH SERVER 1 AND 2 sudo iptables -A INPUT -p tcp -s 112.522.6.61 -m multiport --dports 7000,9042 -m state --state NEW,ESTABLISHED -j ACCEPT sudo iptables -A INPUT -p tcp -s 112.522.6.62 -m multiport --dports 7000,9042 -m state --state NEW,ESTABLISHED -j ACCEPT sudo bash -c \"iptables-save > /etc/iptables/rules.v4\" sudo netfilter-persistent reload sudo nodetool status Dead Node Fix \u00b6 When your server/desktop restarts, you may face a dead node. You need replace the address with the same IP for it to work. cd /etc/cassandra subtl cassandra-env.sh Add the following line in the last row assuming server 1 is dead where the IP is 112.522.6.61 JVM_OPTS=\"$JVM_OPTS -Dcassandra.replace_address=112.522.6.61\" Then run the following bash commands sudo rm -rf /var/lib/cassandra/data/system/* sudo service cassandra restart sudo nodetool status You might have to wait awhile and re-run sudo nodetool status for the DB to get up and running. Summary \u00b6 We have successfully set up a 3-node Cassandra cluster DB after all these steps. We will move on to interacting with the cluster with CQLSH and the Python Driver in subsequent guides. Citation \u00b6 If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Cassandra Cluster Setup"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#setting-up-cassandra-multi-node-cluster","text":"","title":"Setting up Cassandra Multi-node Cluster"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#install-apache-cassandra","text":"Patience is key initially It will be quite tedious to many people in setting up a multi-node Cassandra cluster. You are likely to face a lot of problems. But with a strong community and this tried-and-proven guide, you should be ok! Just be patient. Remember when you're asking for help on StackOverflow, here or anywhere else, do paste the logs to make everyone's life easier to help you debug quickly. All you need to do is to run this base command and paste the logs. cat /var/log/cassandra/system.log Here, we will install all of the required Debian packages. To ensure you've the latest version of Cassandra, please use the official link at cassandra.apache.org/download . Bash Commands echo \"deb http://www.apache.org/dist/cassandra/debian 311x main\" | sudo tee -a /etc/apt/sources.list.d/cassandra.sources.list curl https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add - sudo apt-get update sudo apt-get install cassandra This would get you up and running with Cassandra v3.11.","title":"Install Apache Cassandra"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#install-datastax-python-cassandra-driver","text":"We need to install the Python Cassandra Driver so we can easily interact with our Cassandra Database without using the CQLSH (this is Cassandra's shell to run CQL commands to perform CRUD operations) commandline directly. What is CRUD? CRUD is the acronym of CREATE, READ, UPDATE, and DELETE. For databases, we typically have these 4 categories of operations so we can create new data points, update, read or delete them in our database. Bash Commands pip install cassandra-driver This is strictly for installation on Linux platforms, refer to the official website for more details.","title":"Install DataStax Python Cassandra Driver"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#single-node-cassandra-cluster","text":"Before we venture into the cool world of multi-node cluster requiring many servers, we will start with a single node cluster that only requires a single server (desktop/laptop). Once you've installed everything so far, you should run the following. Bash commands sudo service cassandra start sudo nodetool status And this will print out something like that: Datacenter: datacenter1 ============================= Status=Up/Down |/ State=Normal/Leaving/Joining/Moving -- Address Load Tokens Owns (effective) Host ID Rack UN 127.0.0.1 318.78 KiB 256 100.0% g5ac4c9-99b7-65d-24cfd82524f9 rack1 That's it, we've built our single node Cassandra database.","title":"Single Node Cassandra Cluster"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#multi-node-cassandra-cluster","text":"Remember it's standard to have at least 3 nodes, and in a basic 3 separate server configuration (3 separate desktops for non-enterprise users). Because of this, we will base this tutorial on setting up a 3-node cluster. But the same steps apply if you want to even do 10000 nodes. Installing Sublime critical Sublime is a text editor, and it would help if you're not familiar with VIM which I use frequently. Honestly using Sublime to edit all the following files we will edit is much easier, trust me! To install Sublime run the following bash commands in sequence: wget -qO - https://download.sublimetext.com/sublimehq-pub.gpg | sudo apt-key add - echo \"deb https://download.sublimetext.com/ apt/stable/\" | sudo tee /etc/apt/sources.list.d/sublime-text.list sudo apt-get update sudo apt-get install sublime-text Now check if it works by running this command in your bash: subl This should open Sublime text editor! You're now ready. Before we move on to the steps to set up each node, we need the IP address of all 3 servers. This is simple, just run the following bash command: ifconfig | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\\.){3}[0-9]*).*/\\2/p' This would return the server's IP address that we need for example: server_1_ip: 112.522.6.61 server_2_ip: 112.522.6.62 server_3_ip: 112.522.6.63 We will be using these IPs as a base for configuration in the subsequent sections. Take note yours would differ and you need to change accordingly.","title":"Multi-node Cassandra Cluster"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#steps-per-node","text":"Critical Section You Need To Repeat This is a critcal section. On EVERY server, you need to repeat all the steps shown in this section. In our case of a 3 node cluster, using 3 servers, we need to repeat this 3 times.","title":"Steps Per Node"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#step-1-modify-cassandra-configuration-settings","text":"Run the following bash to edit the configuration file for server 1 (112.522.6.61): cd /etc/cassandra subl cassandra.yaml Now you need to find the following fields and change them accordingly. cluster_name: 'CassandraDBCluster' seed_provider: - class_name: org.apache.cassandra.locator.SimpleSeedProvider parameters: - seeds: \"112.522.6.61, 112.522.6.62, 112.522.6.63\" listen_address: 112.522.6.61 rpc_address: 112.522.6.61 endpoint_snitch: GossipingPropertyFileSnitch auto_bootstrap: true","title":"Step 1: Modify Cassandra Configuration Settings"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#step-2-modify-rack-details","text":"Run the following bash command to edit the rack details: cd /etc/cassandra subl cassandra-rackdc.properties And change the following fields to this: dc=datacenter1","title":"Step 2: Modify Rack Details"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#step-3-check-status","text":"Restart service with the following bash commands: sudo rm -rf /var/lib/cassandra/data/system/* sudo service cassandra restart sudo nodetool status Here you would see an output like this: Datacenter: datacenter ============================= Status=Up/Down |/ State=Normal/Leaving/Joining/Moving -- Address Load Tokens Owns (effective) Host ID Rack UN 112.522.6.61 318.78 KiB 256 100.0% f5c84c9-99b7-45d-8856-24cfd82523f9 rack1 UN 112.522.6.62 206.42 KiB 256 100.0% ac2f24da-1b2c-4e3-8a75-0e28ec366a4 rack1 UN 112.522.6.63 338.34 KiB 256 100.0% 55a227d-ffbe-45d5-8698-60c374b3a6b rack1","title":"Step 3: Check Status"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#step-4-configure-firewall-ip-settings","text":"Run the following bash commands sudo apt-get install iptables-persistent sudo iptables -A INPUT -p tcp -s 112.522.6.62 -m multiport --dports 7000,9042 -m state --state NEW,ESTABLISHED -j ACCEPT sudo iptables -A INPUT -p tcp -s 112.522.6.63 -m multiport --dports 7000,9042 -m state --state NEW,ESTABLISHED -j ACCEPT sudo bash -c \"iptables-save > /etc/iptables/rules.v4\" sudo netfilter-persistent reload sudo nodetool status","title":"Step 4: Configure Firewall IP Settings"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#repeat-steps-1-to-4","text":"You have to repeat all 4 steps for the other 2 servers but step 1 and step 4 requires slightly different settings. For the other servers, you need to basically change the IP to the server's IP. And for the IP firewall settings, you need to allow entry from the other 2 servers instead of itself.","title":"Repeat Steps 1 to 4"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#server-2-example-for-step-2","text":"cluster_name: 'CassandraDBCluster' seed_provider: - class_name: org.apache.cassandra.locator.SimpleSeedProvider parameters: - seeds: \"112.522.6.61, 112.522.6.62, 112.522.6.63\" # ONLY THIS LINE CHANGES listen_address: 112.522.6.62 # ONLY THIS LINE CHANGES rpc_address: 112.522.6.62 endpoint_snitch: GossipingPropertyFileSnitch auto_bootstrap: true","title":"Server 2 Example for Step 2"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#server-3-example-for-step-2","text":"cluster_name: 'CassandraDBCluster' seed_provider: - class_name: org.apache.cassandra.locator.SimpleSeedProvider parameters: - seeds: \"112.522.6.61, 112.522.6.62, 112.522.6.63\" # ONLY THIS LINE CHANGES listen_address: 112.522.6.63 # ONLY THIS LINE CHANGES rpc_address: 112.522.6.63 endpoint_snitch: GossipingPropertyFileSnitch auto_bootstrap: true","title":"Server 3 Example for Step 2"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#server-2-example-for-step-4","text":"sudo apt-get install iptables-persistent # ONLY THESE LINES CHANGE TO ALLOW COMMUNICATION WITH SERVER 1 AND 3 sudo iptables -A INPUT -p tcp -s 112.522.6.61 -m multiport --dports 7000,9042 -m state --state NEW,ESTABLISHED -j ACCEPT sudo iptables -A INPUT -p tcp -s 112.522.6.63 -m multiport --dports 7000,9042 -m state --state NEW,ESTABLISHED -j ACCEPT sudo bash -c \"iptables-save > /etc/iptables/rules.v4\" sudo netfilter-persistent reload sudo nodetool status","title":"Server 2 Example for Step 4"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#server-3-example-for-step-4","text":"sudo apt-get install iptables-persistent # ONLY THESE LINES CHANGE TO ALLOW COMMUNICATION WITH SERVER 1 AND 2 sudo iptables -A INPUT -p tcp -s 112.522.6.61 -m multiport --dports 7000,9042 -m state --state NEW,ESTABLISHED -j ACCEPT sudo iptables -A INPUT -p tcp -s 112.522.6.62 -m multiport --dports 7000,9042 -m state --state NEW,ESTABLISHED -j ACCEPT sudo bash -c \"iptables-save > /etc/iptables/rules.v4\" sudo netfilter-persistent reload sudo nodetool status","title":"Server 3 Example for Step 4"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#dead-node-fix","text":"When your server/desktop restarts, you may face a dead node. You need replace the address with the same IP for it to work. cd /etc/cassandra subtl cassandra-env.sh Add the following line in the last row assuming server 1 is dead where the IP is 112.522.6.61 JVM_OPTS=\"$JVM_OPTS -Dcassandra.replace_address=112.522.6.61\" Then run the following bash commands sudo rm -rf /var/lib/cassandra/data/system/* sudo service cassandra restart sudo nodetool status You might have to wait awhile and re-run sudo nodetool status for the DB to get up and running.","title":"Dead Node Fix"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#summary","text":"We have successfully set up a 3-node Cassandra cluster DB after all these steps. We will move on to interacting with the cluster with CQLSH and the Python Driver in subsequent guides.","title":"Summary"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#citation","text":"If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Citation"},{"location":"deep_learning/course_progression/","text":"Course Progression \u00b6 If you would like a smooth transition in learning deep learning concepts, you need to follow the materials in a sequential order. Some sections are still pending as I am working on them, and they will have the icon beside them. 1. Practical Deep Learning with PyTorch \u00b6 Matrices Gradients Linear Regression Logistic Regression Feedforward Neural Networks (FNN) Convolutional Neural Networks (CNN) Recurrent Neural Networks (RNN) Long Short Term Memory Neural Networks (LSTM) Autoencoders (AE) Fully-connected Overcomplete Autoencoder (AE) Variational Autoencoders (VAE) Adversarial Autoencoders (AAE) Generative Adversarial Networks (GAN) Transformers 2. Improving Deep Learning Models with PyTorch \u00b6 Derivatives, Gradients and Jacobian Gradient Descent and Backpropagation (From Scratch FNN Regression) Learning Rate Scheduling Optimizers Advanced Learning Rate Optimization Weight Initializations and Activation Functions Overfitting Prevention Loss, Accuracy and Weight Visualizations Data Preprocessing for Images and Videos Data Preprocessing for Time Series 3. Deep Reinforcement Learning with PyTorch \u00b6 Supervised Learning to Reinforcement Learning Markov Decision Processes and Bellman Equations Dynamic Programming Monte Carlo Approach Temporal-Difference Policy Gradient: REINFORCE Policy Gradient: Actor-Critic Policy Gradient: A2C/A3C Policy Gradient: ACKTR Policy Gradient: PPO Policy Gradient: DPG Policy Gradient: DDPG (DQN & DPG) 4. From Scratch with Python and PyTorch \u00b6 From Scratch Logistic Regression Classification From Scratch FNN Classification From Scratch CNN Classification From Scratch RNN Classification From Scratch LSTM Classification From Scratch AE 5. Large Language Models with PyTorch \u00b6 Introduction What is Temperature","title":"Course Progression"},{"location":"deep_learning/course_progression/#course-progression","text":"If you would like a smooth transition in learning deep learning concepts, you need to follow the materials in a sequential order. Some sections are still pending as I am working on them, and they will have the icon beside them.","title":"Course Progression"},{"location":"deep_learning/course_progression/#1-practical-deep-learning-with-pytorch","text":"Matrices Gradients Linear Regression Logistic Regression Feedforward Neural Networks (FNN) Convolutional Neural Networks (CNN) Recurrent Neural Networks (RNN) Long Short Term Memory Neural Networks (LSTM) Autoencoders (AE) Fully-connected Overcomplete Autoencoder (AE) Variational Autoencoders (VAE) Adversarial Autoencoders (AAE) Generative Adversarial Networks (GAN) Transformers","title":"1. Practical Deep Learning with PyTorch"},{"location":"deep_learning/course_progression/#2-improving-deep-learning-models-with-pytorch","text":"Derivatives, Gradients and Jacobian Gradient Descent and Backpropagation (From Scratch FNN Regression) Learning Rate Scheduling Optimizers Advanced Learning Rate Optimization Weight Initializations and Activation Functions Overfitting Prevention Loss, Accuracy and Weight Visualizations Data Preprocessing for Images and Videos Data Preprocessing for Time Series","title":"2. Improving Deep Learning Models with PyTorch"},{"location":"deep_learning/course_progression/#3-deep-reinforcement-learning-with-pytorch","text":"Supervised Learning to Reinforcement Learning Markov Decision Processes and Bellman Equations Dynamic Programming Monte Carlo Approach Temporal-Difference Policy Gradient: REINFORCE Policy Gradient: Actor-Critic Policy Gradient: A2C/A3C Policy Gradient: ACKTR Policy Gradient: PPO Policy Gradient: DPG Policy Gradient: DDPG (DQN & DPG)","title":"3. Deep Reinforcement Learning with PyTorch"},{"location":"deep_learning/course_progression/#4-from-scratch-with-python-and-pytorch","text":"From Scratch Logistic Regression Classification From Scratch FNN Classification From Scratch CNN Classification From Scratch RNN Classification From Scratch LSTM Classification From Scratch AE","title":"4. From Scratch with Python and PyTorch"},{"location":"deep_learning/course_progression/#5-large-language-models-with-pytorch","text":"Introduction What is Temperature","title":"5. Large Language Models with PyTorch"},{"location":"deep_learning/intro/","text":"Deep Learning and Deep Reinforcement Learning Theory and Programming Tutorials \u00b6 We'll be covering both CPU and GPU implementations of deep learning and deep reinforcement learning algorithms Packages and Languages you will Learn to Use Python PyTorch NumPy Gym If you would like a more visual and guided experience, feel free to take our video course . Paths \u00b6 Work in progress This open-source portion is still a work in progress, it may be sparse in explanation as traditionally all our explanation are done via video. Stay tuned while we gradually upload our tutorials and notes. Feel free to contact Ritchie Ng if you would like to contribute via our Facebook page. Also take note that these notes are best used as a referral. This is because we have yet to expand it comprehensively to be a stand-alone guide. Go head and take our video course that provides a much easier, proven-to-work, experience. All of our code allows you to run in a notebook for this deep learning section. Please use a jupyter notebook and run the examples from the start of the page to the end. Remember to Right mouse click > Open image in new tab if you would like to zoom into the diagrams if you find them too small. Clarifications These are some clarifications we would like to highlight. When we state linear function , more specifically we meant affine function that comprises a linear function and a constant. We did this initially to make it easier as \"linear function\" was easier to digest. For all diagrams that says valid padding , they refer to no padding such that your output size will be smaller than your input size. For all diagrams that says same padding , they refer to zero padding (padding your input with zeroes) such that your output size will be equal to your input size. Errors to be corrected As we are rapidly prototyping there may be some errors. For these errors stated here, they will be corrected soon. Feel free to report bugs, corrections or improvements on our Github repository . If you did not go through all the materials, you would not be familiar with these, so go through them and come back to review these changes. For all diagrams that says dot product , they refer to matrix product .","title":"Introduction"},{"location":"deep_learning/intro/#deep-learning-and-deep-reinforcement-learning-theory-and-programming-tutorials","text":"We'll be covering both CPU and GPU implementations of deep learning and deep reinforcement learning algorithms Packages and Languages you will Learn to Use Python PyTorch NumPy Gym If you would like a more visual and guided experience, feel free to take our video course .","title":"Deep Learning and Deep Reinforcement Learning Theory and Programming Tutorials"},{"location":"deep_learning/intro/#paths","text":"Work in progress This open-source portion is still a work in progress, it may be sparse in explanation as traditionally all our explanation are done via video. Stay tuned while we gradually upload our tutorials and notes. Feel free to contact Ritchie Ng if you would like to contribute via our Facebook page. Also take note that these notes are best used as a referral. This is because we have yet to expand it comprehensively to be a stand-alone guide. Go head and take our video course that provides a much easier, proven-to-work, experience. All of our code allows you to run in a notebook for this deep learning section. Please use a jupyter notebook and run the examples from the start of the page to the end. Remember to Right mouse click > Open image in new tab if you would like to zoom into the diagrams if you find them too small. Clarifications These are some clarifications we would like to highlight. When we state linear function , more specifically we meant affine function that comprises a linear function and a constant. We did this initially to make it easier as \"linear function\" was easier to digest. For all diagrams that says valid padding , they refer to no padding such that your output size will be smaller than your input size. For all diagrams that says same padding , they refer to zero padding (padding your input with zeroes) such that your output size will be equal to your input size. Errors to be corrected As we are rapidly prototyping there may be some errors. For these errors stated here, they will be corrected soon. Feel free to report bugs, corrections or improvements on our Github repository . If you did not go through all the materials, you would not be familiar with these, so go through them and come back to review these changes. For all diagrams that says dot product , they refer to matrix product .","title":"Paths"},{"location":"deep_learning/readings/","text":"Info This is a list of growing number of papers and implementations I think are interesting. Long Tailed Recognition \u00b6 Large-Scale Long-Tailed Recognition in an Open World Frequently in real world scenario there're new unseen classes or samples within the tail classes This tackles the problem with dynamic embedding to bring associative memory to aid prediction of long-tailed classes The model essentially combines direct image features with embeddings from other classes Better Generalization (Overfitting Prevention or Regularization) \u00b6 Domain Randomization for Transferring Deep Neural Networks from Simulation to the Real World They propose to use domain randomization to train deep learning algorithms on synthetic data and transferring to real-world data The idea is that with sufficient variability in the textures of synthetic data, real-world data becomes another variation of the synthetic data It works surprisingly well and it's a simple technique of varying image textures essentially enabling CNNs to be more robust to variations in image textures Optimization \u00b6 Online Learning Rate Adaptation with Hypergradient Descent Reduces the need for learning rate scheduling for SGD, SGD and nesterov momentum, and Adam Uses the concept of hypergradients (gradients w.r.t. learning rate) obtained via reverse-mode automatic differentiation to dynamically update learning rates in real-time alongside weight updates Little additional computation because just needs just one additional copy of original gradients store in memory Severely under-appreciated paper Network Compression \u00b6 Energy-constrained Compression for Deep Neural Networks via Weighted Sparse Projection and Layer Input Masking More production applications of DNN require low-energy consumption environment like self-driving cars, VR goggles, and drones As such it's critical to optimize DNN not for its primary performance (accuracy etc.) but for its energy consumption performance too In the DNN training, this paper introduces an energy budget constraint on top of other optimization objectives This allows optimization of multiple objectives simultaneously (top-1 accuracy and energy consumption for example) It's done through weighted sparse projection and layer input masking Architecture Search \u00b6 DARTS: Differentiable Architecture Search Neural search algorithm based on gradient descent and continuous relaxation in the architecture space. A good move towards automatic architecture designs of neural networks. EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks Scales all dimensions of a CNN, resolution/depth/width using compound coefficient Uses neural architecture search Network Pruning \u00b6 EigenDamage: Structured Pruning in the Kronecker-Factored Eigenbasis Compared to existing Hessian-based methods, this works on the KFE Reported 10x reduction in model size and 8x reduction in FLOPs on Wide ResNet32 (WRN32) Bayesian Deep Learning \u00b6 Fast and Scalable Bayesian Deep Learning by Weight-Perturbation in Adam Variational Adam (Vadam), an alternative to varianal inference via dropout. Vadam perturbs the network's weights when backpropagating, allowing low computation cost uncertainty estimates. Not as good as dropout in terms of performance, but a good direction for computationally cheaper options. Explainability \u00b6 A Unified Approach to Intepreting Model Predictions Introduces SHAP (SHapley Additive exPlanations) \"SHAP assigns each feature an importance value for a particular prediction\" Higher positive SHAP values (red) = increase the probability of the class Higher negative SHAP values (blue) = decrease the probability of the class Hierarchical interpretations for neural network predictions Given a prediction from the deep neural network, agglomerative contextual decomposition (ACD) produces a hierarchical clusters of input features alongside cluster-wise contribution to the final prediction. The hierarchical clustering is then optimized to identify learned clusters driving the DNN's predictions. Cautious \u00b6 Same Stats, Different Graphs: Generating Datasets with Varied Appearance and Identical Statistics through Simulated Annealing Shows through scatterplots that multiple toy datasets although visually very different can have similar summary statistics like mean, standard deviation and pearson correlation This paper emphasises the need to always visualize your data Visualization \u00b6 Netron Easily visualize your saved deep learning models (PyTorch .pth, TensorFlow .pb, MXNet .model, ONNX, and more) You can even check out each node's documentation quickly in the interface Missing Values \u00b6 BRITS If you face problems in missing data in your time series and you use existing imputation methods, there is an alternative called BRITS where it learns missing values in time series via a bidirectional recurrency dynamical system Correlation \u00b6 DCCA: Deep Canonical Correlation Analysis Learn non-linear complex transformations such that resulting transformed data have high linear correlation Alternative to non-parametric methods like kernel canonical correlation analysis (KCCA) and non-linear extension of canonical correlation analysis (CCA) Shown to learn higher correlation representations than CCA/KCCA Deep Reinforcement Learning \u00b6 An Empirical Analysis of Proximal Policy Optimization with Kronecker-factored Natural Gradients Shows 2 SOTA for deep RL currently (2018 / early 2019): PPO and ACKTR Attempts to combined PPO objective with K-FAC natural gradient optimization: PPOKFAC Does not improve sample complexity, stick with either PPO/ACKTR for now","title":"Readings"},{"location":"deep_learning/readings/#long-tailed-recognition","text":"Large-Scale Long-Tailed Recognition in an Open World Frequently in real world scenario there're new unseen classes or samples within the tail classes This tackles the problem with dynamic embedding to bring associative memory to aid prediction of long-tailed classes The model essentially combines direct image features with embeddings from other classes","title":"Long Tailed Recognition"},{"location":"deep_learning/readings/#better-generalization-overfitting-prevention-or-regularization","text":"Domain Randomization for Transferring Deep Neural Networks from Simulation to the Real World They propose to use domain randomization to train deep learning algorithms on synthetic data and transferring to real-world data The idea is that with sufficient variability in the textures of synthetic data, real-world data becomes another variation of the synthetic data It works surprisingly well and it's a simple technique of varying image textures essentially enabling CNNs to be more robust to variations in image textures","title":"Better Generalization (Overfitting Prevention or Regularization)"},{"location":"deep_learning/readings/#optimization","text":"Online Learning Rate Adaptation with Hypergradient Descent Reduces the need for learning rate scheduling for SGD, SGD and nesterov momentum, and Adam Uses the concept of hypergradients (gradients w.r.t. learning rate) obtained via reverse-mode automatic differentiation to dynamically update learning rates in real-time alongside weight updates Little additional computation because just needs just one additional copy of original gradients store in memory Severely under-appreciated paper","title":"Optimization"},{"location":"deep_learning/readings/#network-compression","text":"Energy-constrained Compression for Deep Neural Networks via Weighted Sparse Projection and Layer Input Masking More production applications of DNN require low-energy consumption environment like self-driving cars, VR goggles, and drones As such it's critical to optimize DNN not for its primary performance (accuracy etc.) but for its energy consumption performance too In the DNN training, this paper introduces an energy budget constraint on top of other optimization objectives This allows optimization of multiple objectives simultaneously (top-1 accuracy and energy consumption for example) It's done through weighted sparse projection and layer input masking","title":"Network Compression"},{"location":"deep_learning/readings/#architecture-search","text":"DARTS: Differentiable Architecture Search Neural search algorithm based on gradient descent and continuous relaxation in the architecture space. A good move towards automatic architecture designs of neural networks. EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks Scales all dimensions of a CNN, resolution/depth/width using compound coefficient Uses neural architecture search","title":"Architecture Search"},{"location":"deep_learning/readings/#network-pruning","text":"EigenDamage: Structured Pruning in the Kronecker-Factored Eigenbasis Compared to existing Hessian-based methods, this works on the KFE Reported 10x reduction in model size and 8x reduction in FLOPs on Wide ResNet32 (WRN32)","title":"Network Pruning"},{"location":"deep_learning/readings/#bayesian-deep-learning","text":"Fast and Scalable Bayesian Deep Learning by Weight-Perturbation in Adam Variational Adam (Vadam), an alternative to varianal inference via dropout. Vadam perturbs the network's weights when backpropagating, allowing low computation cost uncertainty estimates. Not as good as dropout in terms of performance, but a good direction for computationally cheaper options.","title":"Bayesian Deep Learning"},{"location":"deep_learning/readings/#explainability","text":"A Unified Approach to Intepreting Model Predictions Introduces SHAP (SHapley Additive exPlanations) \"SHAP assigns each feature an importance value for a particular prediction\" Higher positive SHAP values (red) = increase the probability of the class Higher negative SHAP values (blue) = decrease the probability of the class Hierarchical interpretations for neural network predictions Given a prediction from the deep neural network, agglomerative contextual decomposition (ACD) produces a hierarchical clusters of input features alongside cluster-wise contribution to the final prediction. The hierarchical clustering is then optimized to identify learned clusters driving the DNN's predictions.","title":"Explainability"},{"location":"deep_learning/readings/#cautious","text":"Same Stats, Different Graphs: Generating Datasets with Varied Appearance and Identical Statistics through Simulated Annealing Shows through scatterplots that multiple toy datasets although visually very different can have similar summary statistics like mean, standard deviation and pearson correlation This paper emphasises the need to always visualize your data","title":"Cautious"},{"location":"deep_learning/readings/#visualization","text":"Netron Easily visualize your saved deep learning models (PyTorch .pth, TensorFlow .pb, MXNet .model, ONNX, and more) You can even check out each node's documentation quickly in the interface","title":"Visualization"},{"location":"deep_learning/readings/#missing-values","text":"BRITS If you face problems in missing data in your time series and you use existing imputation methods, there is an alternative called BRITS where it learns missing values in time series via a bidirectional recurrency dynamical system","title":"Missing Values"},{"location":"deep_learning/readings/#correlation","text":"DCCA: Deep Canonical Correlation Analysis Learn non-linear complex transformations such that resulting transformed data have high linear correlation Alternative to non-parametric methods like kernel canonical correlation analysis (KCCA) and non-linear extension of canonical correlation analysis (CCA) Shown to learn higher correlation representations than CCA/KCCA","title":"Correlation"},{"location":"deep_learning/readings/#deep-reinforcement-learning","text":"An Empirical Analysis of Proximal Policy Optimization with Kronecker-factored Natural Gradients Shows 2 SOTA for deep RL currently (2018 / early 2019): PPO and ACKTR Attempts to combined PPO objective with K-FAC natural gradient optimization: PPOKFAC Does not improve sample complexity, stick with either PPO/ACKTR for now","title":"Deep Reinforcement Learning"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/","text":"Derivative, Gradient and Jacobian \u00b6 Run Jupyter Notebook You can run the code for this section in this jupyter notebook link . Simplified Equation \u00b6 This is the simplified equation we have been using on how we update our parameters to reach good values (good local or global minima) \\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\) \\(\\theta\\) : parameters (our tensors with gradient accumulation abilities) \\(\\eta\\) : learning rate (how fast we want to learn) \\(\\nabla_\\theta\\) : gradients of loss with respect to the model's parameters Even simplier equation in English: parameters = parameters - learning_rate * parameters_gradients This process can be broken down into 2 sequential parts Backpropagation Gradient descent Simplified Equation Breakdown \u00b6 Our simplified equation can be broken down into 2 parts Backpropagation: getting our gradients Our partial derivatives of loss (scalar number) with respect to (w.r.t.) our model's parameters and w.r.t. our input Backpropagation gets us \\(\\nabla_\\theta\\) which is our gradient Gradient descent: using our gradients to update our parameters Somehow, the terms backpropagation and gradient descent are often mixed together. But they're totally different. Gradient descent relates to using our gradients obtained from backpropagation to update our weights. Gradient descent: \\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\) Steps \u00b6 Derivatives Partial Derivatives Gradients Gradient, Jacobian and Generalized Jacobian Differences Backpropagation: computing gradients Gradient descent: using gradients to update parameters Derivative \u00b6 Given a simple cubic equation: \\(f(x) = 2x^3 + 5\\) Calculating the derivative \\(\\frac{df(x)}{dx}\\) is simply calculating the difference in values of \\(y\\) for an extremely small (infinitesimally) change in value of \\(x\\) which is frequently labelled as \\(h\\) \\(\\frac{df(x)}{dx} = \\displaystyle{\\lim_{h \\to 0}} \\frac{f(x + h) - f(x)}{h}\\) \\(\\frac{f(x + h) - f(x)}{h}\\) is the slope formula similar to what you may be familiar with: Change in \\(y\\) over change in \\(x\\) : \\(\\frac{\\Delta y}{\\Delta x}\\) And the derivative is the slope when \\(h \\rightarrow 0\\) , in essence a super teeny small \\(h\\) Let's break down \\(\\frac{df}{dx} = \\displaystyle{\\lim_{h \\to 0}} \\frac{f(x + h) - f(x)}{h}\\) \\(\\displaystyle{\\lim_{h \\to 0} \\frac{f(x + h) - f(x)}{h}}\\) \\(\\displaystyle{\\lim_{h \\to 0} \\frac{(2(x+h)^3 + 5) - 2x^3 + 5}{h}}\\) \\(\\displaystyle{\\lim_{h \\to 0} \\frac{2(x^2 + 2xh + h^2)(x+h) - 2x^3}{h}}\\) \\(\\displaystyle{\\lim_{h \\to 0} \\frac{2(x^3 + 2x^2h + h^3 + x^2h + 2xh^2 + h^3) - 2x^3}{h}}\\) \\(\\displaystyle{\\lim_{h \\to 0}\\frac{2(x^3 + 3x^2h + h^3 + 2xh^2) - 2x^3}{h}}\\) \\(\\displaystyle{\\lim_{h \\to 0} \\frac{6x^2h + h^3 + 2xh^2}{h}}\\) \\(\\displaystyle{\\lim_{h \\to 0} 6x^2 + h^2 + 2xh} = 6x^2\\) Partial Derivative \u00b6 Ok, it's simple to calculate our derivative when we've only one variable in our function. If we've more than one (as with our parameters in our models), we need to calculate our partial derivatives of our function with respect to our variables Given a simple equation \\(f(x, z) = 4x^4z^3\\) , let us get our partial derivatives 2 parts: partial derivative of our function w.r.t. x and z Partial derivative of our function w.r.t. x: \\(\\frac{\\delta f(x, z)}{\\delta x}\\) Let \\(z\\) term be a constant, \\(a\\) \\(f(x, z) = 4x^4a\\) \\(\\frac{\\delta f(x, z)}{\\delta x} = 16x^3a\\) Now we substitute \\(a\\) with our z term, \\(a = z^3\\) \\(\\frac{\\delta f(x, z)}{\\delta x} = 16x^3z^3\\) Partial derivative of our function w.r.t. z: \\(\\frac{\\delta f(x, z)}{\\delta z}\\) Let \\(x\\) term be a constant, \\(a\\) \\(f(x, z) = 4az^3\\) \\(\\frac{\\delta f(x, z)}{\\delta z} = 12az^2\\) Now we substitute \\(a\\) with our \\(x\\) term, \\(a = x^4\\) \\(\\frac{\\delta f(x, z)}{\\delta z} = 12x^4z^2\\) Ta da! We made it, we calculated our partial derivatives of our function w.r.t. the different variables Gradient \u00b6 We can now put all our partial derivatives into a vector of partial derivatives Also called \"gradient\" Represented by \\(\\nabla_{(x,z)}\\) \\(\\nabla_{(x,z)} = \\begin{bmatrix} \\frac{df(x,z)}{dx} \\\\ \\frac{df(x,z)}{dz} \\end{bmatrix} = \\begin{bmatrix} 16x^3z^3 \\\\ 12x^4z^2 \\end{bmatrix}\\) It is critical to note that the term gradient applies for \\(f : \\mathbb{R}^N \\rightarrow \\mathbb{R}\\) Where our function maps a vector input to a scalar output: in deep learning, our loss function that produces a scalar loss Gradient, Jacobian, and Generalized Jacobian \u00b6 In the case where we have non-scalar outputs, these are the right terms of matrices or vectors containing our partial derivatives Gradient: vector input to scalar output \\(f : \\mathbb{R}^N \\rightarrow \\mathbb{R}\\) Jacobian: vector input to vector output \\(f : \\mathbb{R}^N \\rightarrow \\mathbb{R}^M\\) Generalized Jacobian: tensor input to tensor output In this case, a tensor can be any number of dimensions. Summary \u00b6 We've learnt to... Success Calculate derivatives Calculate partial derivatives Get gradients Differentiate the concepts amongst gradients, Jacobian and Generalized Jacobian Now it is time to move on to backpropagation and gradient descent for a simple 1 hidden layer FNN with all these concepts in mind. Citation \u00b6 If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Derivative, Gradient and Jacobian"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/#derivative-gradient-and-jacobian","text":"Run Jupyter Notebook You can run the code for this section in this jupyter notebook link .","title":"Derivative, Gradient and Jacobian"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/#simplified-equation","text":"This is the simplified equation we have been using on how we update our parameters to reach good values (good local or global minima) \\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\) \\(\\theta\\) : parameters (our tensors with gradient accumulation abilities) \\(\\eta\\) : learning rate (how fast we want to learn) \\(\\nabla_\\theta\\) : gradients of loss with respect to the model's parameters Even simplier equation in English: parameters = parameters - learning_rate * parameters_gradients This process can be broken down into 2 sequential parts Backpropagation Gradient descent","title":"Simplified Equation"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/#simplified-equation-breakdown","text":"Our simplified equation can be broken down into 2 parts Backpropagation: getting our gradients Our partial derivatives of loss (scalar number) with respect to (w.r.t.) our model's parameters and w.r.t. our input Backpropagation gets us \\(\\nabla_\\theta\\) which is our gradient Gradient descent: using our gradients to update our parameters Somehow, the terms backpropagation and gradient descent are often mixed together. But they're totally different. Gradient descent relates to using our gradients obtained from backpropagation to update our weights. Gradient descent: \\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\)","title":"Simplified Equation Breakdown"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/#steps","text":"Derivatives Partial Derivatives Gradients Gradient, Jacobian and Generalized Jacobian Differences Backpropagation: computing gradients Gradient descent: using gradients to update parameters","title":"Steps"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/#derivative","text":"Given a simple cubic equation: \\(f(x) = 2x^3 + 5\\) Calculating the derivative \\(\\frac{df(x)}{dx}\\) is simply calculating the difference in values of \\(y\\) for an extremely small (infinitesimally) change in value of \\(x\\) which is frequently labelled as \\(h\\) \\(\\frac{df(x)}{dx} = \\displaystyle{\\lim_{h \\to 0}} \\frac{f(x + h) - f(x)}{h}\\) \\(\\frac{f(x + h) - f(x)}{h}\\) is the slope formula similar to what you may be familiar with: Change in \\(y\\) over change in \\(x\\) : \\(\\frac{\\Delta y}{\\Delta x}\\) And the derivative is the slope when \\(h \\rightarrow 0\\) , in essence a super teeny small \\(h\\) Let's break down \\(\\frac{df}{dx} = \\displaystyle{\\lim_{h \\to 0}} \\frac{f(x + h) - f(x)}{h}\\) \\(\\displaystyle{\\lim_{h \\to 0} \\frac{f(x + h) - f(x)}{h}}\\) \\(\\displaystyle{\\lim_{h \\to 0} \\frac{(2(x+h)^3 + 5) - 2x^3 + 5}{h}}\\) \\(\\displaystyle{\\lim_{h \\to 0} \\frac{2(x^2 + 2xh + h^2)(x+h) - 2x^3}{h}}\\) \\(\\displaystyle{\\lim_{h \\to 0} \\frac{2(x^3 + 2x^2h + h^3 + x^2h + 2xh^2 + h^3) - 2x^3}{h}}\\) \\(\\displaystyle{\\lim_{h \\to 0}\\frac{2(x^3 + 3x^2h + h^3 + 2xh^2) - 2x^3}{h}}\\) \\(\\displaystyle{\\lim_{h \\to 0} \\frac{6x^2h + h^3 + 2xh^2}{h}}\\) \\(\\displaystyle{\\lim_{h \\to 0} 6x^2 + h^2 + 2xh} = 6x^2\\)","title":"Derivative"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/#partial-derivative","text":"Ok, it's simple to calculate our derivative when we've only one variable in our function. If we've more than one (as with our parameters in our models), we need to calculate our partial derivatives of our function with respect to our variables Given a simple equation \\(f(x, z) = 4x^4z^3\\) , let us get our partial derivatives 2 parts: partial derivative of our function w.r.t. x and z Partial derivative of our function w.r.t. x: \\(\\frac{\\delta f(x, z)}{\\delta x}\\) Let \\(z\\) term be a constant, \\(a\\) \\(f(x, z) = 4x^4a\\) \\(\\frac{\\delta f(x, z)}{\\delta x} = 16x^3a\\) Now we substitute \\(a\\) with our z term, \\(a = z^3\\) \\(\\frac{\\delta f(x, z)}{\\delta x} = 16x^3z^3\\) Partial derivative of our function w.r.t. z: \\(\\frac{\\delta f(x, z)}{\\delta z}\\) Let \\(x\\) term be a constant, \\(a\\) \\(f(x, z) = 4az^3\\) \\(\\frac{\\delta f(x, z)}{\\delta z} = 12az^2\\) Now we substitute \\(a\\) with our \\(x\\) term, \\(a = x^4\\) \\(\\frac{\\delta f(x, z)}{\\delta z} = 12x^4z^2\\) Ta da! We made it, we calculated our partial derivatives of our function w.r.t. the different variables","title":"Partial Derivative"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/#gradient","text":"We can now put all our partial derivatives into a vector of partial derivatives Also called \"gradient\" Represented by \\(\\nabla_{(x,z)}\\) \\(\\nabla_{(x,z)} = \\begin{bmatrix} \\frac{df(x,z)}{dx} \\\\ \\frac{df(x,z)}{dz} \\end{bmatrix} = \\begin{bmatrix} 16x^3z^3 \\\\ 12x^4z^2 \\end{bmatrix}\\) It is critical to note that the term gradient applies for \\(f : \\mathbb{R}^N \\rightarrow \\mathbb{R}\\) Where our function maps a vector input to a scalar output: in deep learning, our loss function that produces a scalar loss","title":"Gradient"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/#gradient-jacobian-and-generalized-jacobian","text":"In the case where we have non-scalar outputs, these are the right terms of matrices or vectors containing our partial derivatives Gradient: vector input to scalar output \\(f : \\mathbb{R}^N \\rightarrow \\mathbb{R}\\) Jacobian: vector input to vector output \\(f : \\mathbb{R}^N \\rightarrow \\mathbb{R}^M\\) Generalized Jacobian: tensor input to tensor output In this case, a tensor can be any number of dimensions.","title":"Gradient, Jacobian, and Generalized Jacobian"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/#summary","text":"We've learnt to... Success Calculate derivatives Calculate partial derivatives Get gradients Differentiate the concepts amongst gradients, Jacobian and Generalized Jacobian Now it is time to move on to backpropagation and gradient descent for a simple 1 hidden layer FNN with all these concepts in mind.","title":"Summary"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/#citation","text":"If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Citation"},{"location":"deep_learning/boosting_models_pytorch/forwardpropagation_backpropagation_gradientdescent/","text":"Forwardpropagation, Backpropagation and Gradient Descent with PyTorch \u00b6 Run Jupyter Notebook You can run the code for this section in this jupyter notebook link . Transiting to Backpropagation \u00b6 Let's go back to our simple FNN to put things in perspective Let us ignore non-linearities for now to keep it simpler, but it's just a tiny change subsequently Given a linear transformation on our input (for simplicity instead of an affine transformation that includes a bias): \\(\\hat y = \\theta x\\) \\(\\theta\\) is our parameters \\(x\\) is our input \\(\\hat y\\) is our prediction Then we have our MSE loss function \\(L = \\frac{1}{2} (\\hat y - y)^2\\) We need to calculate our partial derivatives of our loss w.r.t. our parameters to update our parameters: \\(\\nabla_{\\theta} = \\frac{\\delta L}{\\delta \\theta}\\) With chain rule we have \\(\\frac{\\delta L}{\\delta \\theta} = \\frac{\\delta L}{\\delta \\hat y} \\frac{\\delta \\hat y}{\\delta \\theta}\\) \\(\\frac{\\delta L}{\\delta \\hat y} = (\\hat y - y)\\) \\(\\frac{\\delta \\hat y}{\\delta \\theta}\\) is our partial derivatives of \\(y\\) w.r.t. our parameters (our gradient) as we have covered previously Forward Propagation, Backward Propagation and Gradient Descent \u00b6 All right, now let's put together what we have learnt on backpropagation and apply it on a simple feedforward neural network (FNN) Let us assume the following simple FNN architecture and take note that we do not have bias here to keep things simple FNN architecture Linear function: hidden size = 32 Non-linear function: sigmoid Linear function: output size = 1 Non-linear function: sigmoid We will be going through a binary classification problem classifying 2 types of flowers Output size: 1 (represented by 0 or 1 depending on the flower) Input size: 2 (features of the flower) Number of training samples: 100 Load 3-class dataset We want to set a seed to encourage reproducibility so you can match our loss numbers. import torch import torch.nn as nn # Set manual seed torch . manual_seed ( 2 ) Here we want to load our flower classification dataset of 150 samples. There are 2 features, hence the input size would be 150x2. There is no one-hot encoding so the output would not be a size of 150x3 but a size of 150x1. from sklearn import datasets from sklearn import preprocessing iris = datasets . load_iris () X = torch . tensor ( preprocessing . normalize ( iris . data [:, : 2 ]), dtype = torch . float ) y = torch . tensor ( iris . target . reshape ( - 1 , 1 ), dtype = torch . float ) print ( X . size ()) print ( y . size ()) torch . Size ([ 150 , 2 ]) torch . Size ([ 150 , 1 ]) From 3 class dataset to 2 class dataset We only want 2 classes because we want a binary classification problem. As mentioned, there is no one-hot encoding, so each class is represented by 0, 1, or 2. All we need to do is to filter out all samples with a label of 2 to have 2 classes. # We only take 2 classes to make a binary classification problem X = X [: y [ y < 2 ] . size ()[ 0 ]] y = y [: y [ y < 2 ] . size ()[ 0 ]] ```` ``` python print ( X . size ()) print ( y . size ()) torch . Size ([ 100 , 2 ]) torch . Size ([ 100 , 1 ]) Building our FNN model class from scratch class FNN ( nn . Module ): def __init__ ( self , ): super () . __init__ () # Dimensions for input, hidden and output self . input_dim = 2 self . hidden_dim = 32 self . output_dim = 1 # Learning rate definition self . learning_rate = 0.001 # Our parameters (weights) # w1: 2 x 32 self . w1 = torch . randn ( self . input_dim , self . hidden_dim ) # w2: 32 x 1 self . w2 = torch . randn ( self . hidden_dim , self . output_dim ) def sigmoid ( self , s ): return 1 / ( 1 + torch . exp ( - s )) def sigmoid_first_order_derivative ( self , s ): return s * ( 1 - s ) # Forward propagation def forward ( self , X ): # First linear layer self . y1 = torch . matmul ( X , self . w1 ) # 3 X 3 \".dot\" does not broadcast in PyTorch # First non-linearity self . y2 = self . sigmoid ( self . y1 ) # Second linear layer self . y3 = torch . matmul ( self . y2 , self . w2 ) # Second non-linearity y4 = self . sigmoid ( self . y3 ) return y4 # Backward propagation def backward ( self , X , l , y4 ): # Derivative of binary cross entropy cost w.r.t. final output y4 self . dC_dy4 = y4 - l ''' Gradients for w2: partial derivative of cost w.r.t. w2 dC/dw2 ''' self . dy4_dy3 = self . sigmoid_first_order_derivative ( y4 ) self . dy3_dw2 = self . y2 # Y4 delta: dC_dy4 dy4_dy3 self . y4_delta = self . dC_dy4 * self . dy4_dy3 # This is our gradients for w1: dC_dy4 dy4_dy3 dy3_dw2 self . dC_dw2 = torch . matmul ( torch . t ( self . dy3_dw2 ), self . y4_delta ) ''' Gradients for w1: partial derivative of cost w.r.t w1 dC/dw1 ''' self . dy3_dy2 = self . w2 self . dy2_dy1 = self . sigmoid_first_order_derivative ( self . y2 ) # Y2 delta: (dC_dy4 dy4_dy3) dy3_dy2 dy2_dy1 self . y2_delta = torch . matmul ( self . y4_delta , torch . t ( self . dy3_dy2 )) * self . dy2_dy1 # Gradients for w1: (dC_dy4 dy4_dy3) dy3_dy2 dy2_dy1 dy1_dw1 self . dC_dw1 = torch . matmul ( torch . t ( X ), self . y2_delta ) # Gradient descent on the weights from our 2 linear layers self . w1 -= self . learning_rate * self . dC_dw1 self . w2 -= self . learning_rate * self . dC_dw2 def train ( self , X , l ): # Forward propagation y4 = self . forward ( X ) # Backward propagation and gradient descent self . backward ( X , l , y4 ) Training our FNN model # Instantiate our model class and assign it to our model object model = FNN () # Loss list for plotting of loss behaviour loss_lst = [] # Number of times we want our FNN to look at all 100 samples we have, 100 implies looking through 100x num_epochs = 101 # Let's train our model with 100 epochs for epoch in range ( num_epochs ): # Get our predictions y_hat = model ( X ) # Cross entropy loss, remember this can never be negative by nature of the equation # But it does not mean the loss can't be negative for other loss functions cross_entropy_loss = - ( y * torch . log ( y_hat ) + ( 1 - y ) * torch . log ( 1 - y_hat )) # We have to take cross entropy loss over all our samples, 100 in this 2-class iris dataset mean_cross_entropy_loss = torch . mean ( cross_entropy_loss ) . detach () . item () # Print our mean cross entropy loss if epoch % 20 == 0 : print ( 'Epoch {} | Loss: {} ' . format ( epoch , mean_cross_entropy_loss )) loss_lst . append ( mean_cross_entropy_loss ) # (1) Forward propagation: to get our predictions to pass to our cross entropy loss function # (2) Back propagation: get our partial derivatives w.r.t. parameters (gradients) # (3) Gradient Descent: update our weights with our gradients model . train ( X , y ) Epoch 0 | Loss : 0.9228229522705078 Epoch 20 | Loss : 0.6966760754585266 Epoch 40 | Loss : 0.6714916229248047 Epoch 60 | Loss : 0.6686137914657593 Epoch 80 | Loss : 0.666690468788147 Epoch 100 | Loss : 0.6648102402687073 Our loss is decreasing gradually, so it's learning. It has a possibility of reducing to almost 0 (overfitting) with sufficient model capacity (more layers or wider layers). We will explore overfitting and learning rate optimization subsequently. Summary \u00b6 We've learnt... Success The math behind forwardpropagation, backwardpropagation and gradient descent for FNN Implement a basic FNN from scratch with PyTorch Citation \u00b6 If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Forward- and Backward-propagation and Gradient Descent (From Scratch FNN Regression)"},{"location":"deep_learning/boosting_models_pytorch/forwardpropagation_backpropagation_gradientdescent/#forwardpropagation-backpropagation-and-gradient-descent-with-pytorch","text":"Run Jupyter Notebook You can run the code for this section in this jupyter notebook link .","title":"Forwardpropagation, Backpropagation and Gradient Descent with PyTorch"},{"location":"deep_learning/boosting_models_pytorch/forwardpropagation_backpropagation_gradientdescent/#transiting-to-backpropagation","text":"Let's go back to our simple FNN to put things in perspective Let us ignore non-linearities for now to keep it simpler, but it's just a tiny change subsequently Given a linear transformation on our input (for simplicity instead of an affine transformation that includes a bias): \\(\\hat y = \\theta x\\) \\(\\theta\\) is our parameters \\(x\\) is our input \\(\\hat y\\) is our prediction Then we have our MSE loss function \\(L = \\frac{1}{2} (\\hat y - y)^2\\) We need to calculate our partial derivatives of our loss w.r.t. our parameters to update our parameters: \\(\\nabla_{\\theta} = \\frac{\\delta L}{\\delta \\theta}\\) With chain rule we have \\(\\frac{\\delta L}{\\delta \\theta} = \\frac{\\delta L}{\\delta \\hat y} \\frac{\\delta \\hat y}{\\delta \\theta}\\) \\(\\frac{\\delta L}{\\delta \\hat y} = (\\hat y - y)\\) \\(\\frac{\\delta \\hat y}{\\delta \\theta}\\) is our partial derivatives of \\(y\\) w.r.t. our parameters (our gradient) as we have covered previously","title":"Transiting to Backpropagation"},{"location":"deep_learning/boosting_models_pytorch/forwardpropagation_backpropagation_gradientdescent/#forward-propagation-backward-propagation-and-gradient-descent","text":"All right, now let's put together what we have learnt on backpropagation and apply it on a simple feedforward neural network (FNN) Let us assume the following simple FNN architecture and take note that we do not have bias here to keep things simple FNN architecture Linear function: hidden size = 32 Non-linear function: sigmoid Linear function: output size = 1 Non-linear function: sigmoid We will be going through a binary classification problem classifying 2 types of flowers Output size: 1 (represented by 0 or 1 depending on the flower) Input size: 2 (features of the flower) Number of training samples: 100 Load 3-class dataset We want to set a seed to encourage reproducibility so you can match our loss numbers. import torch import torch.nn as nn # Set manual seed torch . manual_seed ( 2 ) Here we want to load our flower classification dataset of 150 samples. There are 2 features, hence the input size would be 150x2. There is no one-hot encoding so the output would not be a size of 150x3 but a size of 150x1. from sklearn import datasets from sklearn import preprocessing iris = datasets . load_iris () X = torch . tensor ( preprocessing . normalize ( iris . data [:, : 2 ]), dtype = torch . float ) y = torch . tensor ( iris . target . reshape ( - 1 , 1 ), dtype = torch . float ) print ( X . size ()) print ( y . size ()) torch . Size ([ 150 , 2 ]) torch . Size ([ 150 , 1 ]) From 3 class dataset to 2 class dataset We only want 2 classes because we want a binary classification problem. As mentioned, there is no one-hot encoding, so each class is represented by 0, 1, or 2. All we need to do is to filter out all samples with a label of 2 to have 2 classes. # We only take 2 classes to make a binary classification problem X = X [: y [ y < 2 ] . size ()[ 0 ]] y = y [: y [ y < 2 ] . size ()[ 0 ]] ```` ``` python print ( X . size ()) print ( y . size ()) torch . Size ([ 100 , 2 ]) torch . Size ([ 100 , 1 ]) Building our FNN model class from scratch class FNN ( nn . Module ): def __init__ ( self , ): super () . __init__ () # Dimensions for input, hidden and output self . input_dim = 2 self . hidden_dim = 32 self . output_dim = 1 # Learning rate definition self . learning_rate = 0.001 # Our parameters (weights) # w1: 2 x 32 self . w1 = torch . randn ( self . input_dim , self . hidden_dim ) # w2: 32 x 1 self . w2 = torch . randn ( self . hidden_dim , self . output_dim ) def sigmoid ( self , s ): return 1 / ( 1 + torch . exp ( - s )) def sigmoid_first_order_derivative ( self , s ): return s * ( 1 - s ) # Forward propagation def forward ( self , X ): # First linear layer self . y1 = torch . matmul ( X , self . w1 ) # 3 X 3 \".dot\" does not broadcast in PyTorch # First non-linearity self . y2 = self . sigmoid ( self . y1 ) # Second linear layer self . y3 = torch . matmul ( self . y2 , self . w2 ) # Second non-linearity y4 = self . sigmoid ( self . y3 ) return y4 # Backward propagation def backward ( self , X , l , y4 ): # Derivative of binary cross entropy cost w.r.t. final output y4 self . dC_dy4 = y4 - l ''' Gradients for w2: partial derivative of cost w.r.t. w2 dC/dw2 ''' self . dy4_dy3 = self . sigmoid_first_order_derivative ( y4 ) self . dy3_dw2 = self . y2 # Y4 delta: dC_dy4 dy4_dy3 self . y4_delta = self . dC_dy4 * self . dy4_dy3 # This is our gradients for w1: dC_dy4 dy4_dy3 dy3_dw2 self . dC_dw2 = torch . matmul ( torch . t ( self . dy3_dw2 ), self . y4_delta ) ''' Gradients for w1: partial derivative of cost w.r.t w1 dC/dw1 ''' self . dy3_dy2 = self . w2 self . dy2_dy1 = self . sigmoid_first_order_derivative ( self . y2 ) # Y2 delta: (dC_dy4 dy4_dy3) dy3_dy2 dy2_dy1 self . y2_delta = torch . matmul ( self . y4_delta , torch . t ( self . dy3_dy2 )) * self . dy2_dy1 # Gradients for w1: (dC_dy4 dy4_dy3) dy3_dy2 dy2_dy1 dy1_dw1 self . dC_dw1 = torch . matmul ( torch . t ( X ), self . y2_delta ) # Gradient descent on the weights from our 2 linear layers self . w1 -= self . learning_rate * self . dC_dw1 self . w2 -= self . learning_rate * self . dC_dw2 def train ( self , X , l ): # Forward propagation y4 = self . forward ( X ) # Backward propagation and gradient descent self . backward ( X , l , y4 ) Training our FNN model # Instantiate our model class and assign it to our model object model = FNN () # Loss list for plotting of loss behaviour loss_lst = [] # Number of times we want our FNN to look at all 100 samples we have, 100 implies looking through 100x num_epochs = 101 # Let's train our model with 100 epochs for epoch in range ( num_epochs ): # Get our predictions y_hat = model ( X ) # Cross entropy loss, remember this can never be negative by nature of the equation # But it does not mean the loss can't be negative for other loss functions cross_entropy_loss = - ( y * torch . log ( y_hat ) + ( 1 - y ) * torch . log ( 1 - y_hat )) # We have to take cross entropy loss over all our samples, 100 in this 2-class iris dataset mean_cross_entropy_loss = torch . mean ( cross_entropy_loss ) . detach () . item () # Print our mean cross entropy loss if epoch % 20 == 0 : print ( 'Epoch {} | Loss: {} ' . format ( epoch , mean_cross_entropy_loss )) loss_lst . append ( mean_cross_entropy_loss ) # (1) Forward propagation: to get our predictions to pass to our cross entropy loss function # (2) Back propagation: get our partial derivatives w.r.t. parameters (gradients) # (3) Gradient Descent: update our weights with our gradients model . train ( X , y ) Epoch 0 | Loss : 0.9228229522705078 Epoch 20 | Loss : 0.6966760754585266 Epoch 40 | Loss : 0.6714916229248047 Epoch 60 | Loss : 0.6686137914657593 Epoch 80 | Loss : 0.666690468788147 Epoch 100 | Loss : 0.6648102402687073 Our loss is decreasing gradually, so it's learning. It has a possibility of reducing to almost 0 (overfitting) with sufficient model capacity (more layers or wider layers). We will explore overfitting and learning rate optimization subsequently.","title":"Forward Propagation, Backward Propagation and Gradient Descent"},{"location":"deep_learning/boosting_models_pytorch/forwardpropagation_backpropagation_gradientdescent/#summary","text":"We've learnt... Success The math behind forwardpropagation, backwardpropagation and gradient descent for FNN Implement a basic FNN from scratch with PyTorch","title":"Summary"},{"location":"deep_learning/boosting_models_pytorch/forwardpropagation_backpropagation_gradientdescent/#citation","text":"If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Citation"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/","text":"Learning Rate Scheduling \u00b6 Run Jupyter Notebook You can run the code for this section in this jupyter notebook link . Optimization Algorithm: Mini-batch Stochastic Gradient Descent (SGD) \u00b6 We will be using mini-batch gradient descent in all our examples here when scheduling our learning rate Combination of batch gradient descent & stochastic gradient descent \\(\\theta = \\theta - \\eta \\cdot \\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\) Characteristics Compute the gradient of the lost function w.r.t. parameters for n sets of training sample (n input and n label) , \\(\\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\) Use this to update our parameters at every iteration Typically in deep learning, some variation of mini-batch gradient is used where the batch size is a hyperparameter to be determined Learning Intuition Recap \u00b6 Learning process Original parameters \\(\\rightarrow\\) given input, get output \\(\\rightarrow\\) compare with labels \\(\\rightarrow\\) get loss with comparison of input/output \\(\\rightarrow\\) get gradients of loss w.r.t parameters \\(\\rightarrow\\) update parameters so model can churn output closer to labels \\(\\rightarrow\\) repeat For a detailed mathematical account of how this works and how to implement from scratch in Python and PyTorch, you can read our forward- and back-propagation and gradient descent post . Learning Rate Pointers \u00b6 Update parameters so model can churn output closer to labels, lower loss \\(\\theta = \\theta - \\eta \\cdot \\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\) If we set \\(\\eta\\) to be a large value \\(\\rightarrow\\) learn too much (rapid learning) Unable to converge to a good local minima (unable to effectively gradually decrease your loss, overshoot the local lowest value) If we set \\(\\eta\\) to be a small value \\(\\rightarrow\\) learn too little (slow learning) May take too long or unable to converge to a good local minima Need for Learning Rate Schedules \u00b6 Benefits Converge faster Higher accuracy Top Basic Learning Rate Schedules \u00b6 Step-wise Decay Reduce on Loss Plateau Decay Step-wise Learning Rate Decay \u00b6 Step-wise Decay: Every Epoch \u00b6 At every epoch, \\(\\eta_t = \\eta_{t-1}\\gamma\\) \\(\\gamma = 0.1\\) Optimization Algorithm 4: SGD Nesterov Modification of SGD Momentum \\(v_t = \\gamma v_{t-1} + \\eta \\cdot \\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\) \\(\\theta = \\theta - v_t\\) Practical example Given \\(\\eta_t = 0.1\\) and $ \\gamma = 0.01$ Epoch 0: \\(\\eta_t = 0.1\\) Epoch 1: \\(\\eta_{t+1} = 0.1 (0.1) = 0.01\\) Epoch 2: \\(\\eta_{t+2} = 0.1 (0.1)^2 = 0.001\\) Epoch n: \\(\\eta_{t+n} = 0.1 (0.1)^n\\) Code for step-wise learning rate decay at every epoch import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) # Where to add a new import from torch.optim.lr_scheduler import StepLR ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS ''' # step_size: at how many multiples of epoch you decay # step_size = 1, after every 1 epoch, new_lr = lr*gamma # step_size = 2, after every 2 epoch, new_lr = lr*gamma # gamma = decaying factor scheduler = StepLR ( optimizer , step_size = 1 , gamma = 0.1 ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): # Decay Learning Rate scheduler . step () # Print Learning Rate print ( 'Epoch:' , epoch , 'LR:' , scheduler . get_lr ()) for i , ( images , labels ) in enumerate ( train_loader ): # Load images images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Epoch : 0 LR : [ 0.1 ] Iteration : 500. Loss : 0.15292978286743164 . Accuracy : 96 Epoch : 1 LR : [ 0.010000000000000002 ] Iteration : 1000. Loss : 0.1207798570394516 . Accuracy : 97 Epoch : 2 LR : [ 0.0010000000000000002 ] Iteration : 1500. Loss : 0.12287932634353638 . Accuracy : 97 Epoch : 3 LR : [ 0.00010000000000000003 ] Iteration : 2000. Loss : 0.05614742264151573 . Accuracy : 97 Epoch : 4 LR : [ 1.0000000000000003e-05 ] Iteration : 2500. Loss : 0.06775809079408646 . Accuracy : 97 Iteration : 3000. Loss : 0.03737065941095352 . Accuracy : 97 Step-wise Decay: Every 2 Epochs \u00b6 At every 2 epoch, \\(\\eta_t = \\eta_{t-1}\\gamma\\) \\(\\gamma = 0.1\\) Optimization Algorithm 4: SGD Nesterov Modification of SGD Momentum \\(v_t = \\gamma v_{t-1} + \\eta \\cdot \\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\) \\(\\theta = \\theta - v_t\\) Practical example Given \\(\\eta_t = 0.1\\) and \\(\\gamma = 0.01\\) Epoch 0: \\(\\eta_t = 0.1\\) Epoch 1: \\(\\eta_{t+1} = 0.1\\) Epoch 2: \\(\\eta_{t+2} = 0.1 (0.1) = 0.01\\) Code for step-wise learning rate decay at every 2 epoch import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) # Where to add a new import from torch.optim.lr_scheduler import StepLR ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS ''' # step_size: at how many multiples of epoch you decay # step_size = 1, after every 2 epoch, new_lr = lr*gamma # step_size = 2, after every 2 epoch, new_lr = lr*gamma # gamma = decaying factor scheduler = StepLR ( optimizer , step_size = 2 , gamma = 0.1 ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): # Decay Learning Rate scheduler . step () # Print Learning Rate print ( 'Epoch:' , epoch , 'LR:' , scheduler . get_lr ()) for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Epoch : 0 LR : [ 0.1 ] Iteration : 500. Loss : 0.15292978286743164 . Accuracy : 96 Epoch : 1 LR : [ 0.1 ] Iteration : 1000. Loss : 0.11253029108047485 . Accuracy : 96 Epoch : 2 LR : [ 0.010000000000000002 ] Iteration : 1500. Loss : 0.14498558640480042 . Accuracy : 97 Epoch : 3 LR : [ 0.010000000000000002 ] Iteration : 2000. Loss : 0.03691177815198898 . Accuracy : 97 Epoch : 4 LR : [ 0.0010000000000000002 ] Iteration : 2500. Loss : 0.03511016443371773 . Accuracy : 97 Iteration : 3000. Loss : 0.029424520209431648 . Accuracy : 97 Step-wise Decay: Every Epoch, Larger Gamma \u00b6 At every epoch, \\(\\eta_t = \\eta_{t-1}\\gamma\\) \\(\\gamma = 0.96\\) Optimization Algorithm 4: SGD Nesterov Modification of SGD Momentum \\(v_t = \\gamma v_{t-1} + \\eta \\cdot \\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\) \\(\\theta = \\theta - v_t\\) Practical example Given \\(\\eta_t = 0.1\\) and \\(\\gamma = 0.96\\) Epoch 1: \\(\\eta_t = 0.1\\) Epoch 2: \\(\\eta_{t+1} = 0.1 (0.96) = 0.096\\) Epoch 3: \\(\\eta_{t+2} = 0.1 (0.96)^2 = 0.092\\) Epoch n: \\(\\eta_{t+n} = 0.1 (0.96)^n\\) Code for step-wise learning rate decay at every epoch with larger gamma import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) # Where to add a new import from torch.optim.lr_scheduler import StepLR ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS ''' # step_size: at how many multiples of epoch you decay # step_size = 1, after every 2 epoch, new_lr = lr*gamma # step_size = 2, after every 2 epoch, new_lr = lr*gamma # gamma = decaying factor scheduler = StepLR ( optimizer , step_size = 2 , gamma = 0.96 ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): # Decay Learning Rate scheduler . step () # Print Learning Rate print ( 'Epoch:' , epoch , 'LR:' , scheduler . get_lr ()) for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Epoch : 0 LR : [ 0.1 ] Iteration : 500. Loss : 0.15292978286743164 . Accuracy : 96 Epoch : 1 LR : [ 0.1 ] Iteration : 1000. Loss : 0.11253029108047485 . Accuracy : 96 Epoch : 2 LR : [ 0.096 ] Iteration : 1500. Loss : 0.11864850670099258 . Accuracy : 97 Epoch : 3 LR : [ 0.096 ] Iteration : 2000. Loss : 0.030942382290959358 . Accuracy : 97 Epoch : 4 LR : [ 0.09216 ] Iteration : 2500. Loss : 0.04521659016609192 . Accuracy : 97 Iteration : 3000. Loss : 0.027839098125696182 . Accuracy : 97 Pointers on Step-wise Decay \u00b6 You would want to decay your LR gradually when you're training more epochs Converge too fast, to a crappy loss/accuracy, if you decay rapidly To decay slower Larger \\(\\gamma\\) Larger interval of decay Reduce on Loss Plateau Decay \u00b6 Reduce on Loss Plateau Decay, Patience=0, Factor=0.1 \u00b6 Reduce learning rate whenever loss plateaus Patience: number of epochs with no improvement after which learning rate will be reduced Patience = 0 Factor: multiplier to decrease learning rate, \\(lr = lr*factor = \\gamma\\) Factor = 0.1 Optimization Algorithm: SGD Nesterov Modification of SGD Momentum \\(v_t = \\gamma v_{t-1} + \\eta \\cdot \\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\) \\(\\theta = \\theta - v_t\\) Code for reduce on loss plateau learning rate decay of factor 0.1 and 0 patience import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) # Where to add a new import from torch.optim.lr_scheduler import ReduceLROnPlateau ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 6000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS ''' # lr = lr * factor # mode='max': look for the maximum validation accuracy to track # patience: number of epochs - 1 where loss plateaus before decreasing LR # patience = 0, after 1 bad epoch, reduce LR # factor = decaying factor scheduler = ReduceLROnPlateau ( optimizer , mode = 'max' , factor = 0.1 , patience = 0 , verbose = True ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions # Without .item(), it is a uint8 tensor which will not work when you pass this number to the scheduler correct += ( predicted == labels ) . sum () . item () accuracy = 100 * correct / total # Print Loss # print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy)) # Decay Learning Rate, pass validation accuracy for tracking at every epoch print ( 'Epoch {} completed' . format ( epoch )) print ( 'Loss: {} . Accuracy: {} ' . format ( loss . item (), accuracy )) print ( '-' * 20 ) scheduler . step ( accuracy ) Epoch 0 completed Loss : 0.17087846994400024 . Accuracy : 96.26 -------------------- Epoch 1 completed Loss : 0.11688263714313507 . Accuracy : 96.96 -------------------- Epoch 2 completed Loss : 0.035437121987342834 . Accuracy : 96.78 -------------------- Epoch 2 : reducing learning rate of group 0 to 1.0000e-02 . Epoch 3 completed Loss : 0.0324370414018631 . Accuracy : 97.7 -------------------- Epoch 4 completed Loss : 0.022194599732756615 . Accuracy : 98.02 -------------------- Epoch 5 completed Loss : 0.007145566865801811 . Accuracy : 98.03 -------------------- Epoch 6 completed Loss : 0.01673538237810135 . Accuracy : 98.05 -------------------- Epoch 7 completed Loss : 0.025424446910619736 . Accuracy : 98.01 -------------------- Epoch 7 : reducing learning rate of group 0 to 1.0000e-03 . Epoch 8 completed Loss : 0.014696130529046059 . Accuracy : 98.05 -------------------- Epoch 8 : reducing learning rate of group 0 to 1.0000e-04 . Epoch 9 completed Loss : 0.00573748117312789 . Accuracy : 98.04 -------------------- Epoch 9 : reducing learning rate of group 0 to 1.0000e-05 . Reduce on Loss Plateau Decay, Patience=0, Factor=0.5 \u00b6 Reduce learning rate whenever loss plateaus Patience: number of epochs with no improvement after which learning rate will be reduced Patience = 0 Factor: multiplier to decrease learning rate, \\(lr = lr*factor = \\gamma\\) Factor = 0.5 Optimization Algorithm 4: SGD Nesterov Modification of SGD Momentum \\(v_t = \\gamma v_{t-1} + \\eta \\cdot \\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\) \\(\\theta = \\theta - v_t\\) Code for reduce on loss plateau learning rate decay with factor 0.5 and 0 patience import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) # Where to add a new import from torch.optim.lr_scheduler import ReduceLROnPlateau ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 6000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS ''' # lr = lr * factor # mode='max': look for the maximum validation accuracy to track # patience: number of epochs - 1 where loss plateaus before decreasing LR # patience = 0, after 1 bad epoch, reduce LR # factor = decaying factor scheduler = ReduceLROnPlateau ( optimizer , mode = 'max' , factor = 0.5 , patience = 0 , verbose = True ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions # Without .item(), it is a uint8 tensor which will not work when you pass this number to the scheduler correct += ( predicted == labels ) . sum () . item () accuracy = 100 * correct / total # Print Loss # print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy)) # Decay Learning Rate, pass validation accuracy for tracking at every epoch print ( 'Epoch {} completed' . format ( epoch )) print ( 'Loss: {} . Accuracy: {} ' . format ( loss . item (), accuracy )) print ( '-' * 20 ) scheduler . step ( accuracy ) Epoch 0 completed Loss : 0.17087846994400024 . Accuracy : 96.26 -------------------- Epoch 1 completed Loss : 0.11688263714313507 . Accuracy : 96.96 -------------------- Epoch 2 completed Loss : 0.035437121987342834 . Accuracy : 96.78 -------------------- Epoch 2 : reducing learning rate of group 0 to 5.0000e-02 . Epoch 3 completed Loss : 0.04893001914024353 . Accuracy : 97.62 -------------------- Epoch 4 completed Loss : 0.020584167912602425 . Accuracy : 97.86 -------------------- Epoch 5 completed Loss : 0.006022400688380003 . Accuracy : 97.95 -------------------- Epoch 6 completed Loss : 0.028374142944812775 . Accuracy : 97.87 -------------------- Epoch 6 : reducing learning rate of group 0 to 2.5000e-02 . Epoch 7 completed Loss : 0.013204765506088734 . Accuracy : 98.0 -------------------- Epoch 8 completed Loss : 0.010137186385691166 . Accuracy : 97.95 -------------------- Epoch 8 : reducing learning rate of group 0 to 1.2500e-02 . Epoch 9 completed Loss : 0.0035198689438402653 . Accuracy : 98.01 -------------------- Pointers on Reduce on Loss Pleateau Decay \u00b6 In these examples, we used patience=1 because we are running few epochs You should look at a larger patience such as 5 if for example you ran 500 epochs. You should experiment with 2 properties Patience Decay factor Summary \u00b6 We've learnt... Success Learning Rate Intuition Update parameters so model can churn output closer to labels Gradual parameter updates Learning Rate Pointers If we set \\(\\eta\\) to be a large value \\(\\rightarrow\\) learn too much (rapid learning) If we set \\(\\eta\\) to be a small value \\(\\rightarrow\\) learn too little (slow learning) Learning Rate Schedules Step-wise Decay Reduce on Loss Plateau Decay Step-wise Decay Every 1 epoch Every 2 epoch Every 1 epoch, larger gamma Step-wise Decay Pointers Decay LR gradually Larger \\(\\gamma\\) Larger interval of decay (increase epoch) Reduce on Loss Plateau Decay Patience=0, Factor=1 Patience=0, Factor=0.5 Pointers on Reduce on Loss Plateau Decay Larger patience with more epochs 2 hyperparameters to experiment Patience Decay factor Citation \u00b6 If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Learning Rate Scheduling"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#learning-rate-scheduling","text":"Run Jupyter Notebook You can run the code for this section in this jupyter notebook link .","title":"Learning Rate Scheduling"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#optimization-algorithm-mini-batch-stochastic-gradient-descent-sgd","text":"We will be using mini-batch gradient descent in all our examples here when scheduling our learning rate Combination of batch gradient descent & stochastic gradient descent \\(\\theta = \\theta - \\eta \\cdot \\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\) Characteristics Compute the gradient of the lost function w.r.t. parameters for n sets of training sample (n input and n label) , \\(\\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\) Use this to update our parameters at every iteration Typically in deep learning, some variation of mini-batch gradient is used where the batch size is a hyperparameter to be determined","title":"Optimization Algorithm: Mini-batch Stochastic Gradient Descent (SGD)"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#learning-intuition-recap","text":"Learning process Original parameters \\(\\rightarrow\\) given input, get output \\(\\rightarrow\\) compare with labels \\(\\rightarrow\\) get loss with comparison of input/output \\(\\rightarrow\\) get gradients of loss w.r.t parameters \\(\\rightarrow\\) update parameters so model can churn output closer to labels \\(\\rightarrow\\) repeat For a detailed mathematical account of how this works and how to implement from scratch in Python and PyTorch, you can read our forward- and back-propagation and gradient descent post .","title":"Learning Intuition Recap"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#learning-rate-pointers","text":"Update parameters so model can churn output closer to labels, lower loss \\(\\theta = \\theta - \\eta \\cdot \\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\) If we set \\(\\eta\\) to be a large value \\(\\rightarrow\\) learn too much (rapid learning) Unable to converge to a good local minima (unable to effectively gradually decrease your loss, overshoot the local lowest value) If we set \\(\\eta\\) to be a small value \\(\\rightarrow\\) learn too little (slow learning) May take too long or unable to converge to a good local minima","title":"Learning Rate Pointers"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#need-for-learning-rate-schedules","text":"Benefits Converge faster Higher accuracy","title":"Need for Learning Rate Schedules"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#top-basic-learning-rate-schedules","text":"Step-wise Decay Reduce on Loss Plateau Decay","title":"Top Basic Learning Rate Schedules"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#step-wise-learning-rate-decay","text":"","title":"Step-wise Learning Rate Decay"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#step-wise-decay-every-epoch","text":"At every epoch, \\(\\eta_t = \\eta_{t-1}\\gamma\\) \\(\\gamma = 0.1\\) Optimization Algorithm 4: SGD Nesterov Modification of SGD Momentum \\(v_t = \\gamma v_{t-1} + \\eta \\cdot \\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\) \\(\\theta = \\theta - v_t\\) Practical example Given \\(\\eta_t = 0.1\\) and $ \\gamma = 0.01$ Epoch 0: \\(\\eta_t = 0.1\\) Epoch 1: \\(\\eta_{t+1} = 0.1 (0.1) = 0.01\\) Epoch 2: \\(\\eta_{t+2} = 0.1 (0.1)^2 = 0.001\\) Epoch n: \\(\\eta_{t+n} = 0.1 (0.1)^n\\) Code for step-wise learning rate decay at every epoch import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) # Where to add a new import from torch.optim.lr_scheduler import StepLR ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS ''' # step_size: at how many multiples of epoch you decay # step_size = 1, after every 1 epoch, new_lr = lr*gamma # step_size = 2, after every 2 epoch, new_lr = lr*gamma # gamma = decaying factor scheduler = StepLR ( optimizer , step_size = 1 , gamma = 0.1 ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): # Decay Learning Rate scheduler . step () # Print Learning Rate print ( 'Epoch:' , epoch , 'LR:' , scheduler . get_lr ()) for i , ( images , labels ) in enumerate ( train_loader ): # Load images images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Epoch : 0 LR : [ 0.1 ] Iteration : 500. Loss : 0.15292978286743164 . Accuracy : 96 Epoch : 1 LR : [ 0.010000000000000002 ] Iteration : 1000. Loss : 0.1207798570394516 . Accuracy : 97 Epoch : 2 LR : [ 0.0010000000000000002 ] Iteration : 1500. Loss : 0.12287932634353638 . Accuracy : 97 Epoch : 3 LR : [ 0.00010000000000000003 ] Iteration : 2000. Loss : 0.05614742264151573 . Accuracy : 97 Epoch : 4 LR : [ 1.0000000000000003e-05 ] Iteration : 2500. Loss : 0.06775809079408646 . Accuracy : 97 Iteration : 3000. Loss : 0.03737065941095352 . Accuracy : 97","title":"Step-wise Decay: Every Epoch"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#step-wise-decay-every-2-epochs","text":"At every 2 epoch, \\(\\eta_t = \\eta_{t-1}\\gamma\\) \\(\\gamma = 0.1\\) Optimization Algorithm 4: SGD Nesterov Modification of SGD Momentum \\(v_t = \\gamma v_{t-1} + \\eta \\cdot \\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\) \\(\\theta = \\theta - v_t\\) Practical example Given \\(\\eta_t = 0.1\\) and \\(\\gamma = 0.01\\) Epoch 0: \\(\\eta_t = 0.1\\) Epoch 1: \\(\\eta_{t+1} = 0.1\\) Epoch 2: \\(\\eta_{t+2} = 0.1 (0.1) = 0.01\\) Code for step-wise learning rate decay at every 2 epoch import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) # Where to add a new import from torch.optim.lr_scheduler import StepLR ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS ''' # step_size: at how many multiples of epoch you decay # step_size = 1, after every 2 epoch, new_lr = lr*gamma # step_size = 2, after every 2 epoch, new_lr = lr*gamma # gamma = decaying factor scheduler = StepLR ( optimizer , step_size = 2 , gamma = 0.1 ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): # Decay Learning Rate scheduler . step () # Print Learning Rate print ( 'Epoch:' , epoch , 'LR:' , scheduler . get_lr ()) for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Epoch : 0 LR : [ 0.1 ] Iteration : 500. Loss : 0.15292978286743164 . Accuracy : 96 Epoch : 1 LR : [ 0.1 ] Iteration : 1000. Loss : 0.11253029108047485 . Accuracy : 96 Epoch : 2 LR : [ 0.010000000000000002 ] Iteration : 1500. Loss : 0.14498558640480042 . Accuracy : 97 Epoch : 3 LR : [ 0.010000000000000002 ] Iteration : 2000. Loss : 0.03691177815198898 . Accuracy : 97 Epoch : 4 LR : [ 0.0010000000000000002 ] Iteration : 2500. Loss : 0.03511016443371773 . Accuracy : 97 Iteration : 3000. Loss : 0.029424520209431648 . Accuracy : 97","title":"Step-wise Decay: Every 2 Epochs"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#step-wise-decay-every-epoch-larger-gamma","text":"At every epoch, \\(\\eta_t = \\eta_{t-1}\\gamma\\) \\(\\gamma = 0.96\\) Optimization Algorithm 4: SGD Nesterov Modification of SGD Momentum \\(v_t = \\gamma v_{t-1} + \\eta \\cdot \\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\) \\(\\theta = \\theta - v_t\\) Practical example Given \\(\\eta_t = 0.1\\) and \\(\\gamma = 0.96\\) Epoch 1: \\(\\eta_t = 0.1\\) Epoch 2: \\(\\eta_{t+1} = 0.1 (0.96) = 0.096\\) Epoch 3: \\(\\eta_{t+2} = 0.1 (0.96)^2 = 0.092\\) Epoch n: \\(\\eta_{t+n} = 0.1 (0.96)^n\\) Code for step-wise learning rate decay at every epoch with larger gamma import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) # Where to add a new import from torch.optim.lr_scheduler import StepLR ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS ''' # step_size: at how many multiples of epoch you decay # step_size = 1, after every 2 epoch, new_lr = lr*gamma # step_size = 2, after every 2 epoch, new_lr = lr*gamma # gamma = decaying factor scheduler = StepLR ( optimizer , step_size = 2 , gamma = 0.96 ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): # Decay Learning Rate scheduler . step () # Print Learning Rate print ( 'Epoch:' , epoch , 'LR:' , scheduler . get_lr ()) for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Epoch : 0 LR : [ 0.1 ] Iteration : 500. Loss : 0.15292978286743164 . Accuracy : 96 Epoch : 1 LR : [ 0.1 ] Iteration : 1000. Loss : 0.11253029108047485 . Accuracy : 96 Epoch : 2 LR : [ 0.096 ] Iteration : 1500. Loss : 0.11864850670099258 . Accuracy : 97 Epoch : 3 LR : [ 0.096 ] Iteration : 2000. Loss : 0.030942382290959358 . Accuracy : 97 Epoch : 4 LR : [ 0.09216 ] Iteration : 2500. Loss : 0.04521659016609192 . Accuracy : 97 Iteration : 3000. Loss : 0.027839098125696182 . Accuracy : 97","title":"Step-wise Decay: Every Epoch, Larger Gamma"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#pointers-on-step-wise-decay","text":"You would want to decay your LR gradually when you're training more epochs Converge too fast, to a crappy loss/accuracy, if you decay rapidly To decay slower Larger \\(\\gamma\\) Larger interval of decay","title":"Pointers on Step-wise Decay"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#reduce-on-loss-plateau-decay","text":"","title":"Reduce on Loss Plateau Decay"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#reduce-on-loss-plateau-decay-patience0-factor01","text":"Reduce learning rate whenever loss plateaus Patience: number of epochs with no improvement after which learning rate will be reduced Patience = 0 Factor: multiplier to decrease learning rate, \\(lr = lr*factor = \\gamma\\) Factor = 0.1 Optimization Algorithm: SGD Nesterov Modification of SGD Momentum \\(v_t = \\gamma v_{t-1} + \\eta \\cdot \\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\) \\(\\theta = \\theta - v_t\\) Code for reduce on loss plateau learning rate decay of factor 0.1 and 0 patience import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) # Where to add a new import from torch.optim.lr_scheduler import ReduceLROnPlateau ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 6000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS ''' # lr = lr * factor # mode='max': look for the maximum validation accuracy to track # patience: number of epochs - 1 where loss plateaus before decreasing LR # patience = 0, after 1 bad epoch, reduce LR # factor = decaying factor scheduler = ReduceLROnPlateau ( optimizer , mode = 'max' , factor = 0.1 , patience = 0 , verbose = True ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions # Without .item(), it is a uint8 tensor which will not work when you pass this number to the scheduler correct += ( predicted == labels ) . sum () . item () accuracy = 100 * correct / total # Print Loss # print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy)) # Decay Learning Rate, pass validation accuracy for tracking at every epoch print ( 'Epoch {} completed' . format ( epoch )) print ( 'Loss: {} . Accuracy: {} ' . format ( loss . item (), accuracy )) print ( '-' * 20 ) scheduler . step ( accuracy ) Epoch 0 completed Loss : 0.17087846994400024 . Accuracy : 96.26 -------------------- Epoch 1 completed Loss : 0.11688263714313507 . Accuracy : 96.96 -------------------- Epoch 2 completed Loss : 0.035437121987342834 . Accuracy : 96.78 -------------------- Epoch 2 : reducing learning rate of group 0 to 1.0000e-02 . Epoch 3 completed Loss : 0.0324370414018631 . Accuracy : 97.7 -------------------- Epoch 4 completed Loss : 0.022194599732756615 . Accuracy : 98.02 -------------------- Epoch 5 completed Loss : 0.007145566865801811 . Accuracy : 98.03 -------------------- Epoch 6 completed Loss : 0.01673538237810135 . Accuracy : 98.05 -------------------- Epoch 7 completed Loss : 0.025424446910619736 . Accuracy : 98.01 -------------------- Epoch 7 : reducing learning rate of group 0 to 1.0000e-03 . Epoch 8 completed Loss : 0.014696130529046059 . Accuracy : 98.05 -------------------- Epoch 8 : reducing learning rate of group 0 to 1.0000e-04 . Epoch 9 completed Loss : 0.00573748117312789 . Accuracy : 98.04 -------------------- Epoch 9 : reducing learning rate of group 0 to 1.0000e-05 .","title":"Reduce on Loss Plateau Decay, Patience=0, Factor=0.1"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#reduce-on-loss-plateau-decay-patience0-factor05","text":"Reduce learning rate whenever loss plateaus Patience: number of epochs with no improvement after which learning rate will be reduced Patience = 0 Factor: multiplier to decrease learning rate, \\(lr = lr*factor = \\gamma\\) Factor = 0.5 Optimization Algorithm 4: SGD Nesterov Modification of SGD Momentum \\(v_t = \\gamma v_{t-1} + \\eta \\cdot \\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\) \\(\\theta = \\theta - v_t\\) Code for reduce on loss plateau learning rate decay with factor 0.5 and 0 patience import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) # Where to add a new import from torch.optim.lr_scheduler import ReduceLROnPlateau ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 6000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS ''' # lr = lr * factor # mode='max': look for the maximum validation accuracy to track # patience: number of epochs - 1 where loss plateaus before decreasing LR # patience = 0, after 1 bad epoch, reduce LR # factor = decaying factor scheduler = ReduceLROnPlateau ( optimizer , mode = 'max' , factor = 0.5 , patience = 0 , verbose = True ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions # Without .item(), it is a uint8 tensor which will not work when you pass this number to the scheduler correct += ( predicted == labels ) . sum () . item () accuracy = 100 * correct / total # Print Loss # print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy)) # Decay Learning Rate, pass validation accuracy for tracking at every epoch print ( 'Epoch {} completed' . format ( epoch )) print ( 'Loss: {} . Accuracy: {} ' . format ( loss . item (), accuracy )) print ( '-' * 20 ) scheduler . step ( accuracy ) Epoch 0 completed Loss : 0.17087846994400024 . Accuracy : 96.26 -------------------- Epoch 1 completed Loss : 0.11688263714313507 . Accuracy : 96.96 -------------------- Epoch 2 completed Loss : 0.035437121987342834 . Accuracy : 96.78 -------------------- Epoch 2 : reducing learning rate of group 0 to 5.0000e-02 . Epoch 3 completed Loss : 0.04893001914024353 . Accuracy : 97.62 -------------------- Epoch 4 completed Loss : 0.020584167912602425 . Accuracy : 97.86 -------------------- Epoch 5 completed Loss : 0.006022400688380003 . Accuracy : 97.95 -------------------- Epoch 6 completed Loss : 0.028374142944812775 . Accuracy : 97.87 -------------------- Epoch 6 : reducing learning rate of group 0 to 2.5000e-02 . Epoch 7 completed Loss : 0.013204765506088734 . Accuracy : 98.0 -------------------- Epoch 8 completed Loss : 0.010137186385691166 . Accuracy : 97.95 -------------------- Epoch 8 : reducing learning rate of group 0 to 1.2500e-02 . Epoch 9 completed Loss : 0.0035198689438402653 . Accuracy : 98.01 --------------------","title":"Reduce on Loss Plateau Decay, Patience=0, Factor=0.5"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#pointers-on-reduce-on-loss-pleateau-decay","text":"In these examples, we used patience=1 because we are running few epochs You should look at a larger patience such as 5 if for example you ran 500 epochs. You should experiment with 2 properties Patience Decay factor","title":"Pointers on Reduce on Loss Pleateau Decay"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#summary","text":"We've learnt... Success Learning Rate Intuition Update parameters so model can churn output closer to labels Gradual parameter updates Learning Rate Pointers If we set \\(\\eta\\) to be a large value \\(\\rightarrow\\) learn too much (rapid learning) If we set \\(\\eta\\) to be a small value \\(\\rightarrow\\) learn too little (slow learning) Learning Rate Schedules Step-wise Decay Reduce on Loss Plateau Decay Step-wise Decay Every 1 epoch Every 2 epoch Every 1 epoch, larger gamma Step-wise Decay Pointers Decay LR gradually Larger \\(\\gamma\\) Larger interval of decay (increase epoch) Reduce on Loss Plateau Decay Patience=0, Factor=1 Patience=0, Factor=0.5 Pointers on Reduce on Loss Plateau Decay Larger patience with more epochs 2 hyperparameters to experiment Patience Decay factor","title":"Summary"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#citation","text":"If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Citation"},{"location":"deep_learning/boosting_models_pytorch/optimizers/","text":"Optimization Algorithms \u00b6 Run Jupyter Notebook You can run the code for this section in this jupyter notebook link . Introduction to Gradient-descent Optimizers \u00b6 Model Recap: 1 Hidden Layer Feedforward Neural Network (ReLU Activation) \u00b6 Steps \u00b6 Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500. Loss: 0.3440718352794647. Accuracy: 91 Iteration: 1000. Loss: 0.2057694047689438. Accuracy: 93 Iteration: 1500. Loss: 0.2646750807762146. Accuracy: 94 Iteration: 2000. Loss: 0.17563636600971222. Accuracy: 94 Iteration: 2500. Loss: 0.1361844837665558. Accuracy: 95 Iteration: 3000. Loss: 0.11089023947715759. Accuracy: 95 Non-Technical Process \u00b6 Convert inputs/labels to variables Clear gradient buffers Get output given inputs Get loss by comparing with labels Get gradients w.r.t. parameters (backpropagation) Update parameters using gradients (gradient descent) parameters = parameters - learning_rate * parameters_gradients REPEAT Why is it called Gradient Descent? \u00b6 Use gradients (calculated through backpropagation) \\(\\rightarrow\\) update parameters to minimize our loss (descent) \\(\\rightarrow\\) better predictive accuracy Mathematical Interpretation of Gradient Descent \u00b6 Model's parameters: \\(\\theta \\in \u211d^d\\) Loss function: \\(J(\\theta)\\) Gradient w.r.t. parameters: \\(\\nabla J(\\theta)\\) Learning rate: \\(\\eta\\) Batch Gradient descent: \\(\\theta = \\theta - \\eta \\cdot \\nabla J(\\theta)\\) Optimization Algorithm 1: Batch Gradient Descent \u00b6 What we've covered so far: batch gradient descent \\(\\theta = \\theta - \\eta \\cdot \\nabla J(\\theta)\\) Characteristics Compute the gradient of the lost function w.r.t. parameters for the entire training data, \\(\\nabla J(\\theta)\\) Use this to update our parameters at every iteration Problems Unable to fit whole datasets in memory Computationally slow as we attempt to compute a large gradient matrix \\(\\rightarrow\\) first order derivative, \\(\\nabla J(\\theta)\\) Conceptually easy to understand \\(\\rightarrow\\) rarely used Optimization Algorithm 2: Stochastic Gradient Descent \u00b6 Modification of batch gradient descent \\(\\theta = \\theta - \\eta \\cdot \\nabla J(\\theta, x^{i}, y^{i})\\) Characteristics Compute the gradient of the lost function w.r.t. parameters for the one set of training sample (1 input and 1 label) , \\(\\nabla J(\\theta, x^{i}, y^{i})\\) Use this to update our parameters at every iteration Benefits Able to fit large datasets Computationally faster \\(\\rightarrow\\) instead gradients w.r.t to the whole training data, we get the gradients w.r.t. training sample Problems Updating very frequently \\(\\rightarrow\\) huge variance in parameter updates \\(\\rightarrow\\) may overshoot local minima Can be solved by carefully decaying your learning rate \\(\\rightarrow\\) take smaller steps in incorporating gradients to improve the parameters Optimization Algorithm 3: Mini-batch Gradient Descent \u00b6 Combination of batch gradient descent & stochastic gradient descent \\(\\theta = \\theta - \\eta \\cdot \\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\) Characteristics Compute the gradient of the lost function w.r.t. parameters for n sets of training sample (n input and n label) , \\(\\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\) Use this to update our parameters at every iteration Benefits Able to fit large datasets Computationally faster \\(\\rightarrow\\) instead gradients w.r.t to the whole training data, we get the gradients w.r.t. training sample Lower variance of parameter updates This is often called SGD in deep learning frameworks .__. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500. Loss: 0.3440718352794647. Accuracy: 91 Iteration: 1000. Loss: 0.2057694047689438. Accuracy: 93 Iteration: 1500. Loss: 0.2646750807762146. Accuracy: 94 Iteration: 2000. Loss: 0.17563636600971222. Accuracy: 94 Iteration: 2500. Loss: 0.1361844837665558. Accuracy: 95 Iteration: 3000. Loss: 0.11089023947715759. Accuracy: 95 Optimization Algorithm 4: SGD Momentum \u00b6 Modification of SGD \\(v_t = \\gamma v_{t-1} + \\eta \\cdot \\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\) \\(\\theta = \\theta - v_t\\) Characteristics Compute the gradient of the lost function w.r.t. parameters for n sets of training sample (n input and n label) , \\(\\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\) Use this to add to the previous update vector \\(v_{t-1}\\) Momentum, usually set to \\(\\gamma = 0.9\\) Parameters updated with update vector, \\(v_t\\) that incorporates previous update vector \\(\\gamma v_{t}\\) increases if gradient same sign/direction as \\(v_{t-1}\\) Gives SGD the push when it is going in the right direction (minimizing loss) Accelerated convergence \\(\\gamma v_{t}\\) decreases if gradient different sign/direction as \\(v_{t-1}\\) Dampens SGD when it is going in a different direction Lower variation in loss minimization Problems It might go the wrong direction (higher loss) \\(\\rightarrow\\) continue to be accelerated to the wrong direction (higher loss) import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500. Loss: 0.16120098531246185. Accuracy: 96 Iteration: 1000. Loss: 0.15727552771568298. Accuracy: 96 Iteration: 1500. Loss: 0.1303034871816635. Accuracy: 96 Iteration: 2000. Loss: 0.022178759798407555. Accuracy: 97 Iteration: 2500. Loss: 0.07027597725391388. Accuracy: 97 Iteration: 3000. Loss: 0.02519878000020981. Accuracy: 97 Optimization Algorithm 4: SGD Nesterov \u00b6 Modification of SGD Momentum \\(v_t = \\gamma v_{t-1} + \\eta \\cdot \\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\) \\(\\theta = \\theta - v_t\\) Characteristics Compute the gradient of the lost function w.r.t. future approximate parameters for n sets of training sample (n input and n label) , \\(\\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\) Use this to add to the previous update vector \\(v_{t-1}\\) Momentum, usually set to \\(\\gamma = 0.9\\) Gradients w.r.t. future approximate parameters \\(\\rightarrow\\) sense of where we will be \\(\\rightarrow\\) anticipate if we are going in the wrong direction in the next step \\(\\rightarrow\\) slow down accordingly import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500. Loss: 0.15292978286743164. Accuracy: 96 Iteration: 1000. Loss: 0.11253029108047485. Accuracy: 96 Iteration: 1500. Loss: 0.11986596137285233. Accuracy: 96 Iteration: 2000. Loss: 0.016192540526390076. Accuracy: 97 Iteration: 2500. Loss: 0.06744947284460068. Accuracy: 97 Iteration: 3000. Loss: 0.03692319989204407. Accuracy: 97 Optimization Algorithm 4: Adam \u00b6 Adaptive Learning Rates \\(m_t = \\beta_1 m_{t-1} + (1 - \\beta_1)g_t\\) Keeping track of decaying gradient Estimate of the mean of gradients \\(v_t = \\beta_2 v_{t-1} + (1 - \\beta_2)g_t^2\\) Keeping track of decaying squared gradient Estimate of the variance of gradients When \\(m_t, v_t\\) initializes as 0, \\(m_t, v_t \\rightarrow 0\\) initially when decay rates small, \\(\\beta_1, \\beta_2 \\rightarrow 1\\) Need to correct this with: \\(\\hat m_t = \\frac{m_t}{1- \\beta_1}\\) \\(\\hat v_t = \\frac{v_t}{1- \\beta_2}\\) \\(\\theta_{t+1} = \\theta_t - \\frac{\\eta}{\\sqrt{\\hat v_t} + \\epsilon}\\hat m_t\\) Default recommended values \\(\\beta_1 = 0.9\\) \\(\\beta_2 = 0.999\\) \\(\\epsilon = 10^{-8}\\) Instead of learning rate \\(\\rightarrow\\) equations account for estimates of mean/variance of gradients to determine the next learning rate import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' # learning_rate = 0.001 optimizer = torch . optim . Adam ( model . parameters ()) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500. Loss: 0.2703690826892853. Accuracy: 93 Iteration: 1000. Loss: 0.15547044575214386. Accuracy: 95 Iteration: 1500. Loss: 0.17266806960105896. Accuracy: 95 Iteration: 2000. Loss: 0.0865858644247055. Accuracy: 96 Iteration: 2500. Loss: 0.07156120240688324. Accuracy: 96 Iteration: 3000. Loss: 0.04664849117398262. Accuracy: 97 Other Adaptive Algorithms \u00b6 Other adaptive algorithms (like Adam, adapting learning rates) Adagrad Adadelta Adamax RMSProp Optimization Algorithm 5: Adagrad \u00b6 import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' # learning_rate = 0.001 optimizer = torch . optim . Adagrad ( model . parameters ()) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500. Loss: 0.2757369875907898. Accuracy: 92 Iteration: 1000. Loss: 0.1992958039045334. Accuracy: 93 Iteration: 1500. Loss: 0.2227272093296051. Accuracy: 94 Iteration: 2000. Loss: 0.18628711998462677. Accuracy: 94 Iteration: 2500. Loss: 0.1470586657524109. Accuracy: 95 Iteration: 3000. Loss: 0.11748368293046951. Accuracy: 95 Optimization Algorithm 6: Adadelta \u00b6 import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' # learning_rate = 0.001 optimizer = torch . optim . Adadelta ( model . parameters ()) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = Variable ( images . view ( - 1 , 28 * 28 )) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . data [ 0 ], accuracy )) Iteration: 500. Loss: 0.26303035020828247. Accuracy: 93.95 Iteration: 1000. Loss: 0.08731874823570251. Accuracy: 95.83 Iteration: 1500. Loss: 0.11502093076705933. Accuracy: 96.87 Iteration: 2000. Loss: 0.03550947830080986. Accuracy: 97.12 Iteration: 2500. Loss: 0.042649827897548676. Accuracy: 97.54 Iteration: 3000. Loss: 0.03061559610068798. Accuracy: 97.45 Optimization Algorithm 6: Adamax \u00b6 import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' # learning_rate = 0.001 optimizer = torch . optim . Adamax ( model . parameters ()) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500. Loss: 0.29930350184440613. Accuracy: 92 Iteration: 1000. Loss: 0.18749120831489563. Accuracy: 93 Iteration: 1500. Loss: 0.21887679398059845. Accuracy: 95 Iteration: 2000. Loss: 0.14390651881694794. Accuracy: 95 Iteration: 2500. Loss: 0.10771607607603073. Accuracy: 96 Iteration: 3000. Loss: 0.0839928686618805. Accuracy: 96 Optimization Algorithm 7: RMSProp \u00b6 import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' # learning_rate = 0.001 optimizer = torch . optim . RMSprop ( model . parameters ()) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500. Loss: 0.25550296902656555. Accuracy: 95 Iteration: 1000. Loss: 0.17357593774795532. Accuracy: 93 Iteration: 1500. Loss: 0.10597744584083557. Accuracy: 96 Iteration: 2000. Loss: 0.03807783126831055. Accuracy: 96 Iteration: 2500. Loss: 0.10654022544622421. Accuracy: 96 Iteration: 3000. Loss: 0.05745543912053108. Accuracy: 96 Summary of Optimization Algorithms Performance \u00b6 SGD: 95.78% SGD Momentum: 97.69% SGD Nesterov: 97.58% Adam: 97.20% Adagrad: 95.51% Adadelta: 97.45% Adamax: 96.58% RMSProp: 97.1% Performance is not definitive here I have used a seed to ensure you can reproduce results here. However, if you change the seed number you would realize that the performance of these optimization algorithms would change. A solution is to run each optimization on many seeds and get the average performance. Then you can compare the mean performance across all optimization algorithms. There are a lot of other factors like how Adam and SGD Momentum may have different ideal starting learning rates and require different learning rate scheduling. But off the hand, SGD and Adam are very robust optimization algorithms that you can rely on. Subsequently, we will look into more advanced optimization algorithms that are based mainly on SGD and Adam. Simple Suggestions \u00b6 Momentum/Nesterov Powerful if we control the learning rate schedule Adam Lazy to control the learning rate schedule Summary \u00b6 We've learnt... Success Recap of 7 step process Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model Step 6 Update parameters using gradients parameters = parameters - learning_rate * parameters_gradients Gradient descent Using gradients (error signals from loss class) to update parameters Mathematical interpretation: \\(\\theta = \\theta - \\eta \\cdot \\nabla J(\\theta)\\) Optimisation Algorithms Batch gradient descent Stochastic gradient descent Mini-batch gradient descent (SGD) SGD + Momentum SGD + Nesterov Adam Other adaptive algorithms: adagrad, adamax, adadelta, RMSProp Recommendations SGD+M SGD+N Adam Citation \u00b6 If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Optimization Algorithms"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithms","text":"Run Jupyter Notebook You can run the code for this section in this jupyter notebook link .","title":"Optimization Algorithms"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#introduction-to-gradient-descent-optimizers","text":"","title":"Introduction to Gradient-descent Optimizers"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#model-recap-1-hidden-layer-feedforward-neural-network-relu-activation","text":"","title":"Model Recap: 1 Hidden Layer Feedforward Neural Network (ReLU Activation)"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#steps","text":"Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500. Loss: 0.3440718352794647. Accuracy: 91 Iteration: 1000. Loss: 0.2057694047689438. Accuracy: 93 Iteration: 1500. Loss: 0.2646750807762146. Accuracy: 94 Iteration: 2000. Loss: 0.17563636600971222. Accuracy: 94 Iteration: 2500. Loss: 0.1361844837665558. Accuracy: 95 Iteration: 3000. Loss: 0.11089023947715759. Accuracy: 95","title":"Steps"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#non-technical-process","text":"Convert inputs/labels to variables Clear gradient buffers Get output given inputs Get loss by comparing with labels Get gradients w.r.t. parameters (backpropagation) Update parameters using gradients (gradient descent) parameters = parameters - learning_rate * parameters_gradients REPEAT","title":"Non-Technical Process"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#why-is-it-called-gradient-descent","text":"Use gradients (calculated through backpropagation) \\(\\rightarrow\\) update parameters to minimize our loss (descent) \\(\\rightarrow\\) better predictive accuracy","title":"Why is it called Gradient Descent?"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#mathematical-interpretation-of-gradient-descent","text":"Model's parameters: \\(\\theta \\in \u211d^d\\) Loss function: \\(J(\\theta)\\) Gradient w.r.t. parameters: \\(\\nabla J(\\theta)\\) Learning rate: \\(\\eta\\) Batch Gradient descent: \\(\\theta = \\theta - \\eta \\cdot \\nabla J(\\theta)\\)","title":"Mathematical Interpretation of Gradient Descent"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithm-1-batch-gradient-descent","text":"What we've covered so far: batch gradient descent \\(\\theta = \\theta - \\eta \\cdot \\nabla J(\\theta)\\) Characteristics Compute the gradient of the lost function w.r.t. parameters for the entire training data, \\(\\nabla J(\\theta)\\) Use this to update our parameters at every iteration Problems Unable to fit whole datasets in memory Computationally slow as we attempt to compute a large gradient matrix \\(\\rightarrow\\) first order derivative, \\(\\nabla J(\\theta)\\) Conceptually easy to understand \\(\\rightarrow\\) rarely used","title":"Optimization Algorithm 1: Batch Gradient Descent"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithm-2-stochastic-gradient-descent","text":"Modification of batch gradient descent \\(\\theta = \\theta - \\eta \\cdot \\nabla J(\\theta, x^{i}, y^{i})\\) Characteristics Compute the gradient of the lost function w.r.t. parameters for the one set of training sample (1 input and 1 label) , \\(\\nabla J(\\theta, x^{i}, y^{i})\\) Use this to update our parameters at every iteration Benefits Able to fit large datasets Computationally faster \\(\\rightarrow\\) instead gradients w.r.t to the whole training data, we get the gradients w.r.t. training sample Problems Updating very frequently \\(\\rightarrow\\) huge variance in parameter updates \\(\\rightarrow\\) may overshoot local minima Can be solved by carefully decaying your learning rate \\(\\rightarrow\\) take smaller steps in incorporating gradients to improve the parameters","title":"Optimization Algorithm 2: Stochastic Gradient Descent"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithm-3-mini-batch-gradient-descent","text":"Combination of batch gradient descent & stochastic gradient descent \\(\\theta = \\theta - \\eta \\cdot \\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\) Characteristics Compute the gradient of the lost function w.r.t. parameters for n sets of training sample (n input and n label) , \\(\\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\) Use this to update our parameters at every iteration Benefits Able to fit large datasets Computationally faster \\(\\rightarrow\\) instead gradients w.r.t to the whole training data, we get the gradients w.r.t. training sample Lower variance of parameter updates This is often called SGD in deep learning frameworks .__. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500. Loss: 0.3440718352794647. Accuracy: 91 Iteration: 1000. Loss: 0.2057694047689438. Accuracy: 93 Iteration: 1500. Loss: 0.2646750807762146. Accuracy: 94 Iteration: 2000. Loss: 0.17563636600971222. Accuracy: 94 Iteration: 2500. Loss: 0.1361844837665558. Accuracy: 95 Iteration: 3000. Loss: 0.11089023947715759. Accuracy: 95","title":"Optimization Algorithm 3: Mini-batch Gradient Descent"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithm-4-sgd-momentum","text":"Modification of SGD \\(v_t = \\gamma v_{t-1} + \\eta \\cdot \\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\) \\(\\theta = \\theta - v_t\\) Characteristics Compute the gradient of the lost function w.r.t. parameters for n sets of training sample (n input and n label) , \\(\\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\) Use this to add to the previous update vector \\(v_{t-1}\\) Momentum, usually set to \\(\\gamma = 0.9\\) Parameters updated with update vector, \\(v_t\\) that incorporates previous update vector \\(\\gamma v_{t}\\) increases if gradient same sign/direction as \\(v_{t-1}\\) Gives SGD the push when it is going in the right direction (minimizing loss) Accelerated convergence \\(\\gamma v_{t}\\) decreases if gradient different sign/direction as \\(v_{t-1}\\) Dampens SGD when it is going in a different direction Lower variation in loss minimization Problems It might go the wrong direction (higher loss) \\(\\rightarrow\\) continue to be accelerated to the wrong direction (higher loss) import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500. Loss: 0.16120098531246185. Accuracy: 96 Iteration: 1000. Loss: 0.15727552771568298. Accuracy: 96 Iteration: 1500. Loss: 0.1303034871816635. Accuracy: 96 Iteration: 2000. Loss: 0.022178759798407555. Accuracy: 97 Iteration: 2500. Loss: 0.07027597725391388. Accuracy: 97 Iteration: 3000. Loss: 0.02519878000020981. Accuracy: 97","title":"Optimization Algorithm 4: SGD Momentum"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithm-4-sgd-nesterov","text":"Modification of SGD Momentum \\(v_t = \\gamma v_{t-1} + \\eta \\cdot \\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\) \\(\\theta = \\theta - v_t\\) Characteristics Compute the gradient of the lost function w.r.t. future approximate parameters for n sets of training sample (n input and n label) , \\(\\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\) Use this to add to the previous update vector \\(v_{t-1}\\) Momentum, usually set to \\(\\gamma = 0.9\\) Gradients w.r.t. future approximate parameters \\(\\rightarrow\\) sense of where we will be \\(\\rightarrow\\) anticipate if we are going in the wrong direction in the next step \\(\\rightarrow\\) slow down accordingly import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500. Loss: 0.15292978286743164. Accuracy: 96 Iteration: 1000. Loss: 0.11253029108047485. Accuracy: 96 Iteration: 1500. Loss: 0.11986596137285233. Accuracy: 96 Iteration: 2000. Loss: 0.016192540526390076. Accuracy: 97 Iteration: 2500. Loss: 0.06744947284460068. Accuracy: 97 Iteration: 3000. Loss: 0.03692319989204407. Accuracy: 97","title":"Optimization Algorithm 4: SGD Nesterov"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithm-4-adam","text":"Adaptive Learning Rates \\(m_t = \\beta_1 m_{t-1} + (1 - \\beta_1)g_t\\) Keeping track of decaying gradient Estimate of the mean of gradients \\(v_t = \\beta_2 v_{t-1} + (1 - \\beta_2)g_t^2\\) Keeping track of decaying squared gradient Estimate of the variance of gradients When \\(m_t, v_t\\) initializes as 0, \\(m_t, v_t \\rightarrow 0\\) initially when decay rates small, \\(\\beta_1, \\beta_2 \\rightarrow 1\\) Need to correct this with: \\(\\hat m_t = \\frac{m_t}{1- \\beta_1}\\) \\(\\hat v_t = \\frac{v_t}{1- \\beta_2}\\) \\(\\theta_{t+1} = \\theta_t - \\frac{\\eta}{\\sqrt{\\hat v_t} + \\epsilon}\\hat m_t\\) Default recommended values \\(\\beta_1 = 0.9\\) \\(\\beta_2 = 0.999\\) \\(\\epsilon = 10^{-8}\\) Instead of learning rate \\(\\rightarrow\\) equations account for estimates of mean/variance of gradients to determine the next learning rate import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' # learning_rate = 0.001 optimizer = torch . optim . Adam ( model . parameters ()) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500. Loss: 0.2703690826892853. Accuracy: 93 Iteration: 1000. Loss: 0.15547044575214386. Accuracy: 95 Iteration: 1500. Loss: 0.17266806960105896. Accuracy: 95 Iteration: 2000. Loss: 0.0865858644247055. Accuracy: 96 Iteration: 2500. Loss: 0.07156120240688324. Accuracy: 96 Iteration: 3000. Loss: 0.04664849117398262. Accuracy: 97","title":"Optimization Algorithm 4: Adam"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#other-adaptive-algorithms","text":"Other adaptive algorithms (like Adam, adapting learning rates) Adagrad Adadelta Adamax RMSProp","title":"Other Adaptive Algorithms"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithm-5-adagrad","text":"import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' # learning_rate = 0.001 optimizer = torch . optim . Adagrad ( model . parameters ()) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500. Loss: 0.2757369875907898. Accuracy: 92 Iteration: 1000. Loss: 0.1992958039045334. Accuracy: 93 Iteration: 1500. Loss: 0.2227272093296051. Accuracy: 94 Iteration: 2000. Loss: 0.18628711998462677. Accuracy: 94 Iteration: 2500. Loss: 0.1470586657524109. Accuracy: 95 Iteration: 3000. Loss: 0.11748368293046951. Accuracy: 95","title":"Optimization Algorithm 5: Adagrad"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithm-6-adadelta","text":"import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' # learning_rate = 0.001 optimizer = torch . optim . Adadelta ( model . parameters ()) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = Variable ( images . view ( - 1 , 28 * 28 )) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . data [ 0 ], accuracy )) Iteration: 500. Loss: 0.26303035020828247. Accuracy: 93.95 Iteration: 1000. Loss: 0.08731874823570251. Accuracy: 95.83 Iteration: 1500. Loss: 0.11502093076705933. Accuracy: 96.87 Iteration: 2000. Loss: 0.03550947830080986. Accuracy: 97.12 Iteration: 2500. Loss: 0.042649827897548676. Accuracy: 97.54 Iteration: 3000. Loss: 0.03061559610068798. Accuracy: 97.45","title":"Optimization Algorithm 6: Adadelta"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithm-6-adamax","text":"import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' # learning_rate = 0.001 optimizer = torch . optim . Adamax ( model . parameters ()) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500. Loss: 0.29930350184440613. Accuracy: 92 Iteration: 1000. Loss: 0.18749120831489563. Accuracy: 93 Iteration: 1500. Loss: 0.21887679398059845. Accuracy: 95 Iteration: 2000. Loss: 0.14390651881694794. Accuracy: 95 Iteration: 2500. Loss: 0.10771607607603073. Accuracy: 96 Iteration: 3000. Loss: 0.0839928686618805. Accuracy: 96","title":"Optimization Algorithm 6: Adamax"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithm-7-rmsprop","text":"import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets # Set seed torch . manual_seed ( 0 ) ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' # learning_rate = 0.001 optimizer = torch . optim . RMSprop ( model . parameters ()) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500. Loss: 0.25550296902656555. Accuracy: 95 Iteration: 1000. Loss: 0.17357593774795532. Accuracy: 93 Iteration: 1500. Loss: 0.10597744584083557. Accuracy: 96 Iteration: 2000. Loss: 0.03807783126831055. Accuracy: 96 Iteration: 2500. Loss: 0.10654022544622421. Accuracy: 96 Iteration: 3000. Loss: 0.05745543912053108. Accuracy: 96","title":"Optimization Algorithm 7: RMSProp"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#summary-of-optimization-algorithms-performance","text":"SGD: 95.78% SGD Momentum: 97.69% SGD Nesterov: 97.58% Adam: 97.20% Adagrad: 95.51% Adadelta: 97.45% Adamax: 96.58% RMSProp: 97.1% Performance is not definitive here I have used a seed to ensure you can reproduce results here. However, if you change the seed number you would realize that the performance of these optimization algorithms would change. A solution is to run each optimization on many seeds and get the average performance. Then you can compare the mean performance across all optimization algorithms. There are a lot of other factors like how Adam and SGD Momentum may have different ideal starting learning rates and require different learning rate scheduling. But off the hand, SGD and Adam are very robust optimization algorithms that you can rely on. Subsequently, we will look into more advanced optimization algorithms that are based mainly on SGD and Adam.","title":"Summary of Optimization Algorithms Performance"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#simple-suggestions","text":"Momentum/Nesterov Powerful if we control the learning rate schedule Adam Lazy to control the learning rate schedule","title":"Simple Suggestions"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#summary","text":"We've learnt... Success Recap of 7 step process Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model Step 6 Update parameters using gradients parameters = parameters - learning_rate * parameters_gradients Gradient descent Using gradients (error signals from loss class) to update parameters Mathematical interpretation: \\(\\theta = \\theta - \\eta \\cdot \\nabla J(\\theta)\\) Optimisation Algorithms Batch gradient descent Stochastic gradient descent Mini-batch gradient descent (SGD) SGD + Momentum SGD + Nesterov Adam Other adaptive algorithms: adagrad, adamax, adadelta, RMSProp Recommendations SGD+M SGD+N Adam","title":"Summary"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#citation","text":"If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Citation"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/","text":"Weight Initializations & Activation Functions \u00b6 Run Jupyter Notebook You can run the code for this section in this jupyter notebook link . Recap of Logistic Regression \u00b6 Recap of Feedforward Neural Network Activation Function \u00b6 Sigmoid (Logistic) \u00b6 \\(\\sigma(x) = \\frac{1}{1 + e^{-x}}\\) Input number \\(\\rightarrow\\) [0, 1] Large negative number \\(\\rightarrow\\) 0 Large positive number \\(\\rightarrow\\) 1 Cons: Activation saturates at 0 or 1 with gradients \\(\\approx\\) 0 No signal to update weights \\(\\rightarrow\\) cannot learn Solution: Have to carefully initialize weights to prevent this import matplotlib.pyplot as plt % matplotlib inline import numpy as np def sigmoid ( x ): a = [] for item in x : a . append ( 1 / ( 1 + np . exp ( - item ))) return a x = np . arange ( - 10. , 10. , 0.2 ) sig = sigmoid ( x ) plt . style . use ( 'ggplot' ) plt . plot ( x , sig , linewidth = 3.0 ) Tanh \u00b6 \\(\\tanh(x) = 2 \\sigma(2x) -1\\) A scaled sigmoid function Input number \\(\\rightarrow\\) [-1, 1] Cons: Activation saturates at 0 or 1 with gradients \\(\\approx\\) 0 No signal to update weights \\(\\rightarrow\\) cannot learn Solution : Have to carefully initialize weights to prevent this x = np . arange ( - 10. , 10. , 0.2 ) tanh = np . dot ( 2 , sigmoid ( np . dot ( 2 , x ))) - 1 plt . plot ( x , tanh , linewidth = 3.0 ) ReLUs \u00b6 \\(f(x) = \\max(0, x)\\) Pros: Accelerates convergence \\(\\rightarrow\\) train faster Less computationally expensive operation compared to Sigmoid/Tanh exponentials Cons: Many ReLU units \"die\" \\(\\rightarrow\\) gradients = 0 forever Solution : careful learning rate and weight initialization choice x = np . arange ( - 10. , 10. , 0.2 ) relu = np . maximum ( x , 0 ) plt . plot ( x , relu , linewidth = 3.0 ) Why do we need weight initializations or new activation functions? \u00b6 To prevent vanishing/exploding gradients Case 1: Sigmoid/Tanh \u00b6 Problem If variance of input too large: gradients = 0 (vanishing gradients) If variance of input too small: linear \\(\\rightarrow\\) gradients = constant value Solutions Want a constant variance of input to achieve non-linearity \\(\\rightarrow\\) unique gradients for unique updates Xavier Initialization (good constant variance for Sigmoid/Tanh) ReLU or Leaky ReLU Case 2: ReLU \u00b6 Solution to Case 1 Regardless of variance of input: gradients = 0 or 1 Problem But those with 0: no updates (\"dead ReLU units\") Has unlimited output size with input > 0 (explodes gradients subsequently) Solutions He Initialization (good constant variance) Leaky ReLU Case 3: Leaky ReLU \u00b6 Solution to Case 2 Solves the 0 signal issue when input < 0 Problem Has unlimited output size with input > 0 (explodes) Solution He Initialization (good constant variance) Summary of weight initialization solutions to activations \u00b6 Tanh/Sigmoid vanishing gradients can be solved with Xavier initialization Good range of constant variance ReLU/Leaky ReLU exploding gradients can be solved with He initialization Good range of constant variance Types of weight intializations \u00b6 Zero Initialization: set all weights to 0 \u00b6 Every neuron in the network computes the same output \\(\\rightarrow\\) computes the same gradient \\(\\rightarrow\\) same parameter updates Normal Initialization: set all weights to random small numbers \u00b6 Every neuron in the network computes different output \\(\\rightarrow\\) computes different gradient \\(\\rightarrow\\) different parameter updates \"Symmetry breaking\" Problem: variance that grows with the number of inputs Lecun Initialization: normalize variance \u00b6 Solves growing variance with the number of inputs \\(\\rightarrow\\) constant variance Look at a simple feedforward neural network Equations for Lecun Initialization \u00b6 \\(Y = AX + B\\) \\(y = a_1x_1 + a_2x_2 + \\cdot + a_n x_n + b\\) \\(Var(y) = Var(a_1x_1 + a_2x_2 + \\cdot + a_n x_n + b)\\) \\(Var(a_i x_i) = E(x_i)^2 Var(a_i) + E(a_i)^2Var(x_i) + Var(a_i)Var(x_i)\\) General term, you might be more familiar with the following \\(Var(XY) = E(X)^2 Var(Y) + E(Y)^2Var(X) + Var(X)Var(Y)\\) \\(E(x_i)\\) : expectation/mean of \\(x_i\\) \\(E(a_i)\\) : expectation/mean of \\(a_i\\) Assuming inputs/weights drawn i.i.d. with Gaussian distribution of mean=0 \\(E(x_i) = E(a_i) = 0\\) \\(Var(a_i x_i) = Var(a_i)Var(x_i)\\) \\(Var(y) = Var(a_1)Var(x_1) + \\cdot + Var(a_n)Var(x_n)\\) Since the bias, b, is a constant, \\(Var(b) = 0\\) Since i.i.d. \\(Var(y) = n \\times Var(a_i)Var(x_i)\\) Since we want constant variance where \\(Var(y) = Var(x_i)\\) \\(1 = nVar(a_i)\\) \\(Var(a_i) = \\frac{1}{n}\\) This is essentially Lecun initialization, from his paper titled \"Efficient Backpropagation\" We draw our weights i.i.d. with mean=0 and variance = \\(\\frac{1}{n}\\) Where \\(n\\) is the number of input units in the weight tensor Improvements to Lecun Intialization \u00b6 They are essentially slight modifications to Lecun'98 initialization Xavier Intialization Works better for layers with Sigmoid activations \\(var(a_i) = \\frac{1}{n_{in} + n_{out}}\\) Where \\(n_{in}\\) and \\(n_{out}\\) are the number of input and output units in the weight tensor respectively Kaiming Initialization Works better for layers with ReLU or LeakyReLU activations \\(var(a_i) = \\frac{2}{n_{in}}\\) Summary of weight initializations \u00b6 Normal Distribution Lecun Normal Distribution Xavier (Glorot) Normal Distribution Kaiming (He) Normal Distribution Weight Initializations with PyTorch \u00b6 Normal Initialization: Tanh Activation \u00b6 import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets from torch.autograd import Variable # Set seed torch . manual_seed ( 0 ) # Scheduler import from torch.optim.lr_scheduler import StepLR ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Linear weight, W, Y = WX + B nn . init . normal_ ( self . fc1 . weight , mean = 0 , std = 1 ) # Non-linearity self . tanh = nn . Tanh () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) nn . init . normal_ ( self . fc2 . weight , mean = 0 , std = 1 ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . tanh ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS ''' # step_size: at how many multiples of epoch you decay # step_size = 1, after every 2 epoch, new_lr = lr*gamma # step_size = 2, after every 2 epoch, new_lr = lr*gamma # gamma = decaying factor scheduler = StepLR ( optimizer , step_size = 1 , gamma = 0.96 ) ''' STEP 8: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): # Decay Learning Rate scheduler . step () # Print Learning Rate print ( 'Epoch:' , epoch , 'LR:' , scheduler . get_lr ()) for i , ( images , labels ) in enumerate ( train_loader ): # Load images as tensors with gradient accumulation abilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted . type ( torch . FloatTensor ) . cpu () == labels . type ( torch . FloatTensor )) . sum () accuracy = 100. * correct . item () / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Epoch: 0 LR: [0.1] Iteration: 500. Loss: 0.5192779302597046. Accuracy: 87.9 Epoch: 1 LR: [0.096] Iteration: 1000. Loss: 0.4060308337211609. Accuracy: 90.15 Epoch: 2 LR: [0.09216] Iteration: 1500. Loss: 0.2880493104457855. Accuracy: 90.71 Epoch: 3 LR: [0.08847359999999999] Iteration: 2000. Loss: 0.23173095285892487. Accuracy: 91.99 Epoch: 4 LR: [0.084934656] Iteration: 2500. Loss: 0.23814399540424347. Accuracy: 92.32 Iteration: 3000. Loss: 0.19513173401355743. Accuracy: 92.55 Lecun Initialization: Tanh Activation \u00b6 By default, PyTorch uses Lecun initialization, so nothing new has to be done here compared to using Normal, Xavier or Kaiming initialization. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets from torch.autograd import Variable # Set seed torch . manual_seed ( 0 ) # Scheduler import from torch.optim.lr_scheduler import StepLR ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . tanh = nn . Tanh () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . tanh ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS ''' # step_size: at how many multiples of epoch you decay # step_size = 1, after every 2 epoch, new_lr = lr*gamma # step_size = 2, after every 2 epoch, new_lr = lr*gamma # gamma = decaying factor scheduler = StepLR ( optimizer , step_size = 1 , gamma = 0.96 ) ''' STEP 8: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): # Decay Learning Rate scheduler . step () # Print Learning Rate print ( 'Epoch:' , epoch , 'LR:' , scheduler . get_lr ()) for i , ( images , labels ) in enumerate ( train_loader ): # Load images as tensors with gradient accumulation abilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted . type ( torch . FloatTensor ) . cpu () == labels . type ( torch . FloatTensor )) . sum () accuracy = 100. * correct . item () / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Epoch: 0 LR: [0.1] Iteration: 500. Loss: 0.20123475790023804. Accuracy: 95.63 Epoch: 1 LR: [0.096] Iteration: 1000. Loss: 0.10885068774223328. Accuracy: 96.48 Epoch: 2 LR: [0.09216] Iteration: 1500. Loss: 0.1296212077140808. Accuracy: 97.22 Epoch: 3 LR: [0.08847359999999999] Iteration: 2000. Loss: 0.05178885534405708. Accuracy: 97.36 Epoch: 4 LR: [0.084934656] Iteration: 2500. Loss: 0.02619408629834652. Accuracy: 97.61 Iteration: 3000. Loss: 0.02096685953438282. Accuracy: 97.7 Xavier Initialization: Tanh Activation \u00b6 import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets from torch.autograd import Variable # Set seed torch . manual_seed ( 0 ) # Scheduler import from torch.optim.lr_scheduler import StepLR ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Linear weight, W, Y = WX + B nn . init . xavier_normal_ ( self . fc1 . weight ) # Non-linearity self . tanh = nn . Tanh () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) nn . init . xavier_normal_ ( self . fc2 . weight ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . tanh ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS ''' # step_size: at how many multiples of epoch you decay # step_size = 1, after every 2 epoch, new_lr = lr*gamma # step_size = 2, after every 2 epoch, new_lr = lr*gamma # gamma = decaying factor scheduler = StepLR ( optimizer , step_size = 1 , gamma = 0.96 ) ''' STEP 8: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): # Decay Learning Rate scheduler . step () # Print Learning Rate print ( 'Epoch:' , epoch , 'LR:' , scheduler . get_lr ()) for i , ( images , labels ) in enumerate ( train_loader ): # Load images as tensors with gradient accumulation abilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted . type ( torch . FloatTensor ) . cpu () == labels . type ( torch . FloatTensor )) . sum () accuracy = 100. * correct . item () / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Epoch: 0 LR: [0.1] Iteration: 500. Loss: 0.14800140261650085. Accuracy: 95.43 Epoch: 1 LR: [0.096] Iteration: 1000. Loss: 0.17138008773326874. Accuracy: 96.58 Epoch: 2 LR: [0.09216] Iteration: 1500. Loss: 0.07987994700670242. Accuracy: 96.95 Epoch: 3 LR: [0.08847359999999999] Iteration: 2000. Loss: 0.07756654918193817. Accuracy: 97.23 Epoch: 4 LR: [0.084934656] Iteration: 2500. Loss: 0.05563584715127945. Accuracy: 97.6 Iteration: 3000. Loss: 0.07122127711772919. Accuracy: 97.49 Xavier Initialization: ReLU Activation \u00b6 import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets from torch.autograd import Variable # Set seed torch . manual_seed ( 0 ) # Scheduler import from torch.optim.lr_scheduler import StepLR ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Linear weight, W, Y = WX + B nn . init . xavier_normal_ ( self . fc1 . weight ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) nn . init . xavier_normal_ ( self . fc2 . weight ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS ''' # step_size: at how many multiples of epoch you decay # step_size = 1, after every 2 epoch, new_lr = lr*gamma # step_size = 2, after every 2 epoch, new_lr = lr*gamma # gamma = decaying factor scheduler = StepLR ( optimizer , step_size = 1 , gamma = 0.96 ) ''' STEP 8: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): # Decay Learning Rate scheduler . step () # Print Learning Rate print ( 'Epoch:' , epoch , 'LR:' , scheduler . get_lr ()) for i , ( images , labels ) in enumerate ( train_loader ): # Load images as tensors with gradient accumulation abilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted . type ( torch . FloatTensor ) . cpu () == labels . type ( torch . FloatTensor )) . sum () accuracy = 100. * correct . item () / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Epoch: 0 LR: [0.1] Iteration: 500. Loss: 0.1245984435081482. Accuracy: 95.82 Epoch: 1 LR: [0.096] Iteration: 1000. Loss: 0.14348150789737701. Accuracy: 96.72 Epoch: 2 LR: [0.09216] Iteration: 1500. Loss: 0.10421314090490341. Accuracy: 97.3 Epoch: 3 LR: [0.08847359999999999] Iteration: 2000. Loss: 0.04693891853094101. Accuracy: 97.29 Epoch: 4 LR: [0.084934656] Iteration: 2500. Loss: 0.06869587302207947. Accuracy: 97.61 Iteration: 3000. Loss: 0.056865859776735306. Accuracy: 97.48 He Initialization: ReLU Activation \u00b6 import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets from torch.autograd import Variable # Set seed torch . manual_seed ( 0 ) # Scheduler import from torch.optim.lr_scheduler import StepLR ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Linear weight, W, Y = WX + B nn . init . kaiming_normal_ ( self . fc1 . weight ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) nn . init . kaiming_normal_ ( self . fc2 . weight ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS ''' # step_size: at how many multiples of epoch you decay # step_size = 1, after every 2 epoch, new_lr = lr*gamma # step_size = 2, after every 2 epoch, new_lr = lr*gamma # gamma = decaying factor scheduler = StepLR ( optimizer , step_size = 1 , gamma = 0.96 ) ''' STEP 8: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): # Decay Learning Rate scheduler . step () # Print Learning Rate print ( 'Epoch:' , epoch , 'LR:' , scheduler . get_lr ()) for i , ( images , labels ) in enumerate ( train_loader ): # Load images as tensors with gradient accumulation abilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted . type ( torch . FloatTensor ) . cpu () == labels . type ( torch . FloatTensor )) . sum () accuracy = 100. * correct . item () / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Epoch: 0 LR: [0.1] Iteration: 500. Loss: 0.11658752709627151. Accuracy: 95.7 Epoch: 1 LR: [0.096] Iteration: 1000. Loss: 0.15525035560131073. Accuracy: 96.65 Epoch: 2 LR: [0.09216] Iteration: 1500. Loss: 0.09970294684171677. Accuracy: 97.07 Epoch: 3 LR: [0.08847359999999999] Iteration: 2000. Loss: 0.04063304886221886. Accuracy: 97.23 Epoch: 4 LR: [0.084934656] Iteration: 2500. Loss: 0.0719323456287384. Accuracy: 97.7 Iteration: 3000. Loss: 0.04470040276646614. Accuracy: 97.39 Initialization Performance \u00b6 Initialization: Activation Test Accuracy Normal: Tanh 92.55 Lecun: Tanh 97.7 Xavier: Tanh 97.49 Xavier: ReLU 97.48 He: ReLU 97.39 Interpreting the Validation Accuracy Table Take note that these numbers would fluctuate slightly when you change seeds. However, the key point here is that all the other intializations are clearly much better than a basic normal distribution. Whether He, Xavier, or Lecun intialization is better or any other initializations depends on the overall model's architecture (RNN/LSTM/CNN/FNN etc.), activation functions (ReLU, Sigmoid, Tanh etc.) and more. For example, more advanced initializations we will cover subsequently is orthogonal initialization that works better for RNN/LSTM. But due to the math involved in that, we will be covering such advanced initializations in a separate section. Summary \u00b6 We've learnt... Success Recap of LG Recap of FNN Recap of Activation Functions Sigmoid (Logistic) Tanh ReLU Need for Weight Initializations Sigmoid/Tanh: vanishing gradients Constant Variance initialization with Lecun or Xavier ReLU: exploding gradients with dead units He Initialization Leaky ReLU: exploding gradients only He Initialization Types of weight initialisations Zero Normal: growing weight variance Lecun: constant variance Xavier: constant variance for Sigmoid/Tanh Kaiming He: constant variance for ReLU activations PyTorch implementation Citation \u00b6 If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Weight Initialization and Activation Functions"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#weight-initializations-activation-functions","text":"Run Jupyter Notebook You can run the code for this section in this jupyter notebook link .","title":"Weight Initializations &amp; Activation Functions"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#recap-of-logistic-regression","text":"","title":"Recap of Logistic Regression"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#recap-of-feedforward-neural-network-activation-function","text":"","title":"Recap of Feedforward Neural Network Activation Function"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#sigmoid-logistic","text":"\\(\\sigma(x) = \\frac{1}{1 + e^{-x}}\\) Input number \\(\\rightarrow\\) [0, 1] Large negative number \\(\\rightarrow\\) 0 Large positive number \\(\\rightarrow\\) 1 Cons: Activation saturates at 0 or 1 with gradients \\(\\approx\\) 0 No signal to update weights \\(\\rightarrow\\) cannot learn Solution: Have to carefully initialize weights to prevent this import matplotlib.pyplot as plt % matplotlib inline import numpy as np def sigmoid ( x ): a = [] for item in x : a . append ( 1 / ( 1 + np . exp ( - item ))) return a x = np . arange ( - 10. , 10. , 0.2 ) sig = sigmoid ( x ) plt . style . use ( 'ggplot' ) plt . plot ( x , sig , linewidth = 3.0 )","title":"Sigmoid (Logistic)"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#tanh","text":"\\(\\tanh(x) = 2 \\sigma(2x) -1\\) A scaled sigmoid function Input number \\(\\rightarrow\\) [-1, 1] Cons: Activation saturates at 0 or 1 with gradients \\(\\approx\\) 0 No signal to update weights \\(\\rightarrow\\) cannot learn Solution : Have to carefully initialize weights to prevent this x = np . arange ( - 10. , 10. , 0.2 ) tanh = np . dot ( 2 , sigmoid ( np . dot ( 2 , x ))) - 1 plt . plot ( x , tanh , linewidth = 3.0 )","title":"Tanh"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#relus","text":"\\(f(x) = \\max(0, x)\\) Pros: Accelerates convergence \\(\\rightarrow\\) train faster Less computationally expensive operation compared to Sigmoid/Tanh exponentials Cons: Many ReLU units \"die\" \\(\\rightarrow\\) gradients = 0 forever Solution : careful learning rate and weight initialization choice x = np . arange ( - 10. , 10. , 0.2 ) relu = np . maximum ( x , 0 ) plt . plot ( x , relu , linewidth = 3.0 )","title":"ReLUs"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#why-do-we-need-weight-initializations-or-new-activation-functions","text":"To prevent vanishing/exploding gradients","title":"Why do we need weight initializations or new activation functions?"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#case-1-sigmoidtanh","text":"Problem If variance of input too large: gradients = 0 (vanishing gradients) If variance of input too small: linear \\(\\rightarrow\\) gradients = constant value Solutions Want a constant variance of input to achieve non-linearity \\(\\rightarrow\\) unique gradients for unique updates Xavier Initialization (good constant variance for Sigmoid/Tanh) ReLU or Leaky ReLU","title":"Case 1: Sigmoid/Tanh"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#case-2-relu","text":"Solution to Case 1 Regardless of variance of input: gradients = 0 or 1 Problem But those with 0: no updates (\"dead ReLU units\") Has unlimited output size with input > 0 (explodes gradients subsequently) Solutions He Initialization (good constant variance) Leaky ReLU","title":"Case 2: ReLU"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#case-3-leaky-relu","text":"Solution to Case 2 Solves the 0 signal issue when input < 0 Problem Has unlimited output size with input > 0 (explodes) Solution He Initialization (good constant variance)","title":"Case 3: Leaky ReLU"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#summary-of-weight-initialization-solutions-to-activations","text":"Tanh/Sigmoid vanishing gradients can be solved with Xavier initialization Good range of constant variance ReLU/Leaky ReLU exploding gradients can be solved with He initialization Good range of constant variance","title":"Summary of weight initialization solutions to activations"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#types-of-weight-intializations","text":"","title":"Types of weight intializations"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#zero-initialization-set-all-weights-to-0","text":"Every neuron in the network computes the same output \\(\\rightarrow\\) computes the same gradient \\(\\rightarrow\\) same parameter updates","title":"Zero Initialization: set all weights to 0"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#normal-initialization-set-all-weights-to-random-small-numbers","text":"Every neuron in the network computes different output \\(\\rightarrow\\) computes different gradient \\(\\rightarrow\\) different parameter updates \"Symmetry breaking\" Problem: variance that grows with the number of inputs","title":"Normal Initialization: set all weights to random small numbers"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#lecun-initialization-normalize-variance","text":"Solves growing variance with the number of inputs \\(\\rightarrow\\) constant variance Look at a simple feedforward neural network","title":"Lecun Initialization: normalize variance"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#equations-for-lecun-initialization","text":"\\(Y = AX + B\\) \\(y = a_1x_1 + a_2x_2 + \\cdot + a_n x_n + b\\) \\(Var(y) = Var(a_1x_1 + a_2x_2 + \\cdot + a_n x_n + b)\\) \\(Var(a_i x_i) = E(x_i)^2 Var(a_i) + E(a_i)^2Var(x_i) + Var(a_i)Var(x_i)\\) General term, you might be more familiar with the following \\(Var(XY) = E(X)^2 Var(Y) + E(Y)^2Var(X) + Var(X)Var(Y)\\) \\(E(x_i)\\) : expectation/mean of \\(x_i\\) \\(E(a_i)\\) : expectation/mean of \\(a_i\\) Assuming inputs/weights drawn i.i.d. with Gaussian distribution of mean=0 \\(E(x_i) = E(a_i) = 0\\) \\(Var(a_i x_i) = Var(a_i)Var(x_i)\\) \\(Var(y) = Var(a_1)Var(x_1) + \\cdot + Var(a_n)Var(x_n)\\) Since the bias, b, is a constant, \\(Var(b) = 0\\) Since i.i.d. \\(Var(y) = n \\times Var(a_i)Var(x_i)\\) Since we want constant variance where \\(Var(y) = Var(x_i)\\) \\(1 = nVar(a_i)\\) \\(Var(a_i) = \\frac{1}{n}\\) This is essentially Lecun initialization, from his paper titled \"Efficient Backpropagation\" We draw our weights i.i.d. with mean=0 and variance = \\(\\frac{1}{n}\\) Where \\(n\\) is the number of input units in the weight tensor","title":"Equations for Lecun Initialization"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#improvements-to-lecun-intialization","text":"They are essentially slight modifications to Lecun'98 initialization Xavier Intialization Works better for layers with Sigmoid activations \\(var(a_i) = \\frac{1}{n_{in} + n_{out}}\\) Where \\(n_{in}\\) and \\(n_{out}\\) are the number of input and output units in the weight tensor respectively Kaiming Initialization Works better for layers with ReLU or LeakyReLU activations \\(var(a_i) = \\frac{2}{n_{in}}\\)","title":"Improvements to Lecun Intialization"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#summary-of-weight-initializations","text":"Normal Distribution Lecun Normal Distribution Xavier (Glorot) Normal Distribution Kaiming (He) Normal Distribution","title":"Summary of  weight initializations"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#weight-initializations-with-pytorch","text":"","title":"Weight Initializations with PyTorch"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#normal-initialization-tanh-activation","text":"import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets from torch.autograd import Variable # Set seed torch . manual_seed ( 0 ) # Scheduler import from torch.optim.lr_scheduler import StepLR ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Linear weight, W, Y = WX + B nn . init . normal_ ( self . fc1 . weight , mean = 0 , std = 1 ) # Non-linearity self . tanh = nn . Tanh () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) nn . init . normal_ ( self . fc2 . weight , mean = 0 , std = 1 ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . tanh ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS ''' # step_size: at how many multiples of epoch you decay # step_size = 1, after every 2 epoch, new_lr = lr*gamma # step_size = 2, after every 2 epoch, new_lr = lr*gamma # gamma = decaying factor scheduler = StepLR ( optimizer , step_size = 1 , gamma = 0.96 ) ''' STEP 8: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): # Decay Learning Rate scheduler . step () # Print Learning Rate print ( 'Epoch:' , epoch , 'LR:' , scheduler . get_lr ()) for i , ( images , labels ) in enumerate ( train_loader ): # Load images as tensors with gradient accumulation abilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted . type ( torch . FloatTensor ) . cpu () == labels . type ( torch . FloatTensor )) . sum () accuracy = 100. * correct . item () / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Epoch: 0 LR: [0.1] Iteration: 500. Loss: 0.5192779302597046. Accuracy: 87.9 Epoch: 1 LR: [0.096] Iteration: 1000. Loss: 0.4060308337211609. Accuracy: 90.15 Epoch: 2 LR: [0.09216] Iteration: 1500. Loss: 0.2880493104457855. Accuracy: 90.71 Epoch: 3 LR: [0.08847359999999999] Iteration: 2000. Loss: 0.23173095285892487. Accuracy: 91.99 Epoch: 4 LR: [0.084934656] Iteration: 2500. Loss: 0.23814399540424347. Accuracy: 92.32 Iteration: 3000. Loss: 0.19513173401355743. Accuracy: 92.55","title":"Normal Initialization: Tanh Activation"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#lecun-initialization-tanh-activation","text":"By default, PyTorch uses Lecun initialization, so nothing new has to be done here compared to using Normal, Xavier or Kaiming initialization. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets from torch.autograd import Variable # Set seed torch . manual_seed ( 0 ) # Scheduler import from torch.optim.lr_scheduler import StepLR ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . tanh = nn . Tanh () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . tanh ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS ''' # step_size: at how many multiples of epoch you decay # step_size = 1, after every 2 epoch, new_lr = lr*gamma # step_size = 2, after every 2 epoch, new_lr = lr*gamma # gamma = decaying factor scheduler = StepLR ( optimizer , step_size = 1 , gamma = 0.96 ) ''' STEP 8: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): # Decay Learning Rate scheduler . step () # Print Learning Rate print ( 'Epoch:' , epoch , 'LR:' , scheduler . get_lr ()) for i , ( images , labels ) in enumerate ( train_loader ): # Load images as tensors with gradient accumulation abilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted . type ( torch . FloatTensor ) . cpu () == labels . type ( torch . FloatTensor )) . sum () accuracy = 100. * correct . item () / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Epoch: 0 LR: [0.1] Iteration: 500. Loss: 0.20123475790023804. Accuracy: 95.63 Epoch: 1 LR: [0.096] Iteration: 1000. Loss: 0.10885068774223328. Accuracy: 96.48 Epoch: 2 LR: [0.09216] Iteration: 1500. Loss: 0.1296212077140808. Accuracy: 97.22 Epoch: 3 LR: [0.08847359999999999] Iteration: 2000. Loss: 0.05178885534405708. Accuracy: 97.36 Epoch: 4 LR: [0.084934656] Iteration: 2500. Loss: 0.02619408629834652. Accuracy: 97.61 Iteration: 3000. Loss: 0.02096685953438282. Accuracy: 97.7","title":"Lecun Initialization: Tanh Activation"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#xavier-initialization-tanh-activation","text":"import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets from torch.autograd import Variable # Set seed torch . manual_seed ( 0 ) # Scheduler import from torch.optim.lr_scheduler import StepLR ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Linear weight, W, Y = WX + B nn . init . xavier_normal_ ( self . fc1 . weight ) # Non-linearity self . tanh = nn . Tanh () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) nn . init . xavier_normal_ ( self . fc2 . weight ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . tanh ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS ''' # step_size: at how many multiples of epoch you decay # step_size = 1, after every 2 epoch, new_lr = lr*gamma # step_size = 2, after every 2 epoch, new_lr = lr*gamma # gamma = decaying factor scheduler = StepLR ( optimizer , step_size = 1 , gamma = 0.96 ) ''' STEP 8: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): # Decay Learning Rate scheduler . step () # Print Learning Rate print ( 'Epoch:' , epoch , 'LR:' , scheduler . get_lr ()) for i , ( images , labels ) in enumerate ( train_loader ): # Load images as tensors with gradient accumulation abilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted . type ( torch . FloatTensor ) . cpu () == labels . type ( torch . FloatTensor )) . sum () accuracy = 100. * correct . item () / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Epoch: 0 LR: [0.1] Iteration: 500. Loss: 0.14800140261650085. Accuracy: 95.43 Epoch: 1 LR: [0.096] Iteration: 1000. Loss: 0.17138008773326874. Accuracy: 96.58 Epoch: 2 LR: [0.09216] Iteration: 1500. Loss: 0.07987994700670242. Accuracy: 96.95 Epoch: 3 LR: [0.08847359999999999] Iteration: 2000. Loss: 0.07756654918193817. Accuracy: 97.23 Epoch: 4 LR: [0.084934656] Iteration: 2500. Loss: 0.05563584715127945. Accuracy: 97.6 Iteration: 3000. Loss: 0.07122127711772919. Accuracy: 97.49","title":"Xavier Initialization: Tanh Activation"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#xavier-initialization-relu-activation","text":"import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets from torch.autograd import Variable # Set seed torch . manual_seed ( 0 ) # Scheduler import from torch.optim.lr_scheduler import StepLR ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Linear weight, W, Y = WX + B nn . init . xavier_normal_ ( self . fc1 . weight ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) nn . init . xavier_normal_ ( self . fc2 . weight ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS ''' # step_size: at how many multiples of epoch you decay # step_size = 1, after every 2 epoch, new_lr = lr*gamma # step_size = 2, after every 2 epoch, new_lr = lr*gamma # gamma = decaying factor scheduler = StepLR ( optimizer , step_size = 1 , gamma = 0.96 ) ''' STEP 8: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): # Decay Learning Rate scheduler . step () # Print Learning Rate print ( 'Epoch:' , epoch , 'LR:' , scheduler . get_lr ()) for i , ( images , labels ) in enumerate ( train_loader ): # Load images as tensors with gradient accumulation abilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted . type ( torch . FloatTensor ) . cpu () == labels . type ( torch . FloatTensor )) . sum () accuracy = 100. * correct . item () / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Epoch: 0 LR: [0.1] Iteration: 500. Loss: 0.1245984435081482. Accuracy: 95.82 Epoch: 1 LR: [0.096] Iteration: 1000. Loss: 0.14348150789737701. Accuracy: 96.72 Epoch: 2 LR: [0.09216] Iteration: 1500. Loss: 0.10421314090490341. Accuracy: 97.3 Epoch: 3 LR: [0.08847359999999999] Iteration: 2000. Loss: 0.04693891853094101. Accuracy: 97.29 Epoch: 4 LR: [0.084934656] Iteration: 2500. Loss: 0.06869587302207947. Accuracy: 97.61 Iteration: 3000. Loss: 0.056865859776735306. Accuracy: 97.48","title":"Xavier Initialization: ReLU Activation"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#he-initialization-relu-activation","text":"import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets from torch.autograd import Variable # Set seed torch . manual_seed ( 0 ) # Scheduler import from torch.optim.lr_scheduler import StepLR ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Linear weight, W, Y = WX + B nn . init . kaiming_normal_ ( self . fc1 . weight ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) nn . init . kaiming_normal_ ( self . fc2 . weight ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate , momentum = 0.9 , nesterov = True ) ''' STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS ''' # step_size: at how many multiples of epoch you decay # step_size = 1, after every 2 epoch, new_lr = lr*gamma # step_size = 2, after every 2 epoch, new_lr = lr*gamma # gamma = decaying factor scheduler = StepLR ( optimizer , step_size = 1 , gamma = 0.96 ) ''' STEP 8: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): # Decay Learning Rate scheduler . step () # Print Learning Rate print ( 'Epoch:' , epoch , 'LR:' , scheduler . get_lr ()) for i , ( images , labels ) in enumerate ( train_loader ): # Load images as tensors with gradient accumulation abilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted . type ( torch . FloatTensor ) . cpu () == labels . type ( torch . FloatTensor )) . sum () accuracy = 100. * correct . item () / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Epoch: 0 LR: [0.1] Iteration: 500. Loss: 0.11658752709627151. Accuracy: 95.7 Epoch: 1 LR: [0.096] Iteration: 1000. Loss: 0.15525035560131073. Accuracy: 96.65 Epoch: 2 LR: [0.09216] Iteration: 1500. Loss: 0.09970294684171677. Accuracy: 97.07 Epoch: 3 LR: [0.08847359999999999] Iteration: 2000. Loss: 0.04063304886221886. Accuracy: 97.23 Epoch: 4 LR: [0.084934656] Iteration: 2500. Loss: 0.0719323456287384. Accuracy: 97.7 Iteration: 3000. Loss: 0.04470040276646614. Accuracy: 97.39","title":"He Initialization: ReLU Activation"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#initialization-performance","text":"Initialization: Activation Test Accuracy Normal: Tanh 92.55 Lecun: Tanh 97.7 Xavier: Tanh 97.49 Xavier: ReLU 97.48 He: ReLU 97.39 Interpreting the Validation Accuracy Table Take note that these numbers would fluctuate slightly when you change seeds. However, the key point here is that all the other intializations are clearly much better than a basic normal distribution. Whether He, Xavier, or Lecun intialization is better or any other initializations depends on the overall model's architecture (RNN/LSTM/CNN/FNN etc.), activation functions (ReLU, Sigmoid, Tanh etc.) and more. For example, more advanced initializations we will cover subsequently is orthogonal initialization that works better for RNN/LSTM. But due to the math involved in that, we will be covering such advanced initializations in a separate section.","title":"Initialization Performance"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#summary","text":"We've learnt... Success Recap of LG Recap of FNN Recap of Activation Functions Sigmoid (Logistic) Tanh ReLU Need for Weight Initializations Sigmoid/Tanh: vanishing gradients Constant Variance initialization with Lecun or Xavier ReLU: exploding gradients with dead units He Initialization Leaky ReLU: exploding gradients only He Initialization Types of weight initialisations Zero Normal: growing weight variance Lecun: constant variance Xavier: constant variance for Sigmoid/Tanh Kaiming He: constant variance for ReLU activations PyTorch implementation","title":"Summary"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#citation","text":"If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Citation"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/","text":"Markov Decision Processes (MDPs) \u00b6 Typically we can frame all RL tasks as MDPs 1 Intuitively, it's sort of a way to frame RL tasks such that we can solve them in a \"principled\" manner. We will go into the specifics throughout this tutorial The key in MDPs is the Markov Property Essentially the future depends on the present and not the past More specifically, the future is independent of the past given the present There's an assumption the present state encapsulates past information. This is not always true, see the note below. Putting into the context of what we have covered so far: our agent can (1) control its action based on its current (2) completely known state Back to the \"driving to avoid puppy\" example: given we know there is a dog in front of the car as the current state and the car is always moving forward (no reverse driving), the agent can decide to take a left/right turn to avoid colliding with the puppy in front Two main characteristics for MDPs \u00b6 Control over state transitions States completely observable Other Markov Models Permutations of whether there is presence of the two main characteristics would lead to different Markov models. These are not important now, but it gives you an idea of what other frameworks we can use besides MDPs. Types of Markov Models \u00b6 {++Control++} over state transitions and {++completely observable++} states: MDPs {++Control++} over state transitions and partially observable states : Partially Observable MDPs (POMDPs) No control over state transitions and {++completely observable++} states: Markov Chain No control over state transitions and partially observable states: Hidden Markov Model POMDPs \u00b6 Imagine our driving example where we don't know if the car is going forward/backward in its state, but only know there is a puppy in the center lane in front, this is a partially observable state There are ways to counter this Use the complete history to construct the current state Represent the current state as a probability distribution ( bayesian approach ) of what the agent perceives of the current state Using a RNN to form the current state that encapsulates the past 2 5 Components of MDPs \u00b6 \\(\\mathcal{S}\\) : set of states \\(\\mathcal{A}\\) : set of actions \\(\\mathcal{R}\\) : reward function \\(\\mathcal{P}\\) : transition probability function \\(\\gamma\\) : discount for future rewards Remembering 5 components with a mnemonic A mnemonic I use to remember the 5 components is the acronym \"SARPY\" (sar-py). I know \\(\\gamma\\) is not \\(\\mathcal{Y}\\) but it looks like a y so there's that. Moving From MDPs to Optimal Policy \u00b6 We have an agent acting in an environment The way the environment reacts to the agent's actions ( \\(a\\) ) is dictated by a model The agent can take actions ( \\(a\\) ) to move from one state ( \\(s\\) ) to another new state ( \\(s')\\) When the agent has transited to a new state ( \\(s')\\) , there will a reward ( \\(r\\) ) We may or may not know our model Model-based RL : this is where we can {++clearly define++} our (1) transition probabilities and/or (2) reward function A global minima can be attained via Dynamic Programming (DP) Model-free RL : this is where we cannot clearly define our (1) transition probabilities and/or (2) reward function Most real-world problems are under this category so we will mostly place our attention on this category How the agent acts ( \\(a\\) ) in its current state ( \\(s\\) ) is specified by its policy ( \\(\\pi(s)\\) ) It can either be deterministic or stochastic Deterministic policy : \\(a = \\pi(s)\\) Stochastic policy : \\(\\mathbb{P}_\\pi [A=a \\vert S=s] = \\pi(a | s)\\) This is the proability of taking an action given the current state under the policy \\(\\mathcal{A}\\) : set of all actions \\(\\mathcal{S}\\) : set of all states When the agent acts given its state under the policy ( \\(\\pi(a | s)\\) ) , the transition probability function \\(\\mathcal{P}\\) determines the subsequent state ( \\(s'\\) ) \\(\\mathcal{P}_{ss'}^a = \\mathcal{P}(s' \\vert s, a) = \\mathbb{P} [S_{t+1} = s' \\vert S_t = s, A_t = a]\\) When the agent act based on its policy ( \\(\\pi(a | s)\\) ) and transited to the new state determined by the transition probability function \\(\\mathcal{P}_{ss'}^a\\) it gets a reward based on the reward function as a feedback \\(\\mathcal{R}_s^a = \\mathbb{E} [\\mathcal{R}_{t+1} \\vert S_t = s, A_t = a]\\) Rewards are short-term, given as feedback after the agent takes an action and transits to a new state. Summing all future rewards and discounting them would lead to our return \\(\\mathcal{G}\\) \\(\\mathcal{G}_t = \\sum_{i=0}^{N} \\gamma^k \\mathcal{R}_{t+1+i}\\) \\(\\gamma\\) , our discount factor which ranges from 0 to 1 (inclusive) reduces the weightage of future rewards allowing us to balance between short-term and long-term goals With our return \\(\\mathcal{G}\\) , we then have our state-value function \\(\\mathcal{V}_{\\pi}\\) (how good to stay in that state) and our action-value or q-value function \\(\\mathcal{Q}_{\\pi}\\) (how good to take the action) \\(\\mathcal{V}_{\\pi}(s) = \\mathbb{E}_{\\pi}[\\mathcal{G}_t \\vert \\mathcal{S}_t = s]\\) \\(\\mathcal{Q}_{\\pi}(s, a) = \\mathbb{E}_{\\pi}[\\mathcal{G}_t \\vert \\mathcal{S}_t = s, \\mathcal{A}_t = a]\\) The advantage function is simply the difference between the two functions \\(\\mathcal{A}_{\\pi}(s, a) = \\mathcal{Q}_{\\pi}(s, a) - \\mathcal{V}_{\\pi}(s)\\) Seems useless at this stage, but this advantage function will be used in some key algorithms we are covering Since our policy determines how our agent acts given its state, achieving an optimal policy \\(\\pi_*\\) would mean achieving optimal actions that is exactly what we want! Basic Categories of Approaches We've covered state-value functions, action-value functions, model-free RL and model-based RL. They form general overarching categories of how we design our agent. Categories of Design \u00b6 State-value based: search for the optimal state-value function (goodness of action in the state) Action-value based: search for the optimal action-value function (goodness of policy) Actor-critic based: using both state-value and action-value function Model based: attempts to model the environment to find the best policy Model-free based: trial and error to optimize for the best policy to get the most rewards instead of modelling the environment explicitly Optimal Policy \u00b6 Optimal policy \\(\\pi_*\\) \u2192 optimal state-value and action-value functions \u2192 max return \u2192 argmax of value functions \\(\\pi_{*} = \\arg\\max_{\\pi} \\mathcal{V}_{\\pi}(s) = \\arg\\max_{\\pi} \\mathcal{Q}_{\\pi}(s, a)\\) To calculate argmax of value functions \u2192 we need max return \\(\\mathcal{G}_t\\) \u2192 need max sum of rewards \\(\\mathcal{R}_s^a\\) To get max sum of rewards \\(\\mathcal{R}_s^a\\) we will rely on the Bellman Equations. 3 Bellman Equation \u00b6 Essentially, the Bellman Equation breaks down our value functions into two parts Immediate reward Discounted future value function State-value function can be broken into: \\(\\begin{aligned} \\mathcal{V}_{\\pi}(s) &= \\mathbb{E}[\\mathcal{G}_t \\vert \\mathcal{S}_t = s] \\\\ &= \\mathbb{E} [\\mathcal{R}_{t+1} + \\gamma \\mathcal{R}_{t+2} + \\gamma^2 \\mathcal{R}_{t+3} + \\dots \\vert \\mathcal{S}_t = s] \\\\ &= \\mathbb{E} [\\mathcal{R}_{t+1} + \\gamma (\\mathcal{R}_{t+2} + \\gamma \\mathcal{R}_{t+3} + \\dots) \\vert \\mathcal{S}_t = s] \\\\ &= \\mathbb{E} [\\mathcal{R}_{t+1} + \\gamma \\mathcal{G}_{t+1} \\vert \\mathcal{S}_t = s] \\\\ &= \\mathbb{E} [\\mathcal{R}_{t+1} + \\gamma \\mathcal{V}_{\\pi}(\\mathcal{s}_{t+1}) \\vert \\mathcal{S}_t = s] \\end{aligned}\\) Action-value function can be broken into: \\(\\mathcal{Q}_{\\pi}(s, a) = \\mathbb{E} [\\mathcal{R}_{t+1} + \\gamma \\mathcal{Q}_{\\pi}(\\mathcal{s}_{t+1}, \\mathcal{a}_{t+1}) \\vert \\mathcal{S}_t = s, \\mathcal{A} = a]\\) Key Recap on Value Functions \u00b6 \\(\\mathcal{V}_{\\pi}(s) = \\mathbb{E}_{\\pi}[\\mathcal{G}_t \\vert \\mathcal{S}_t = s]\\) State-value function: tells us how good to be in that state \\(\\mathcal{Q}_{\\pi}(s, a) = \\mathbb{E}_{\\pi}[\\mathcal{G}_t \\vert \\mathcal{S}_t = s, \\mathcal{A}_t = a]\\) Action-value function: tells us how good to take actions given state Bellman Expectation Equations \u00b6 Now we can move from Bellman Equations into Bellman Expectation Equations Basic: State-value function \\(\\mathcal{V}_{\\pi}(s)\\) Current state \\(\\mathcal{S}\\) Multiple possible actions determined by stochastic policy \\(\\pi(a | s)\\) Each possible action is associated with a action-value function \\(\\mathcal{Q}_{\\pi}(s, a)\\) returning a value of that particular action Multiplying the possible actions with the action-value function and summing them gives us an indication of how good it is to be in that state Final equation: \\(\\mathcal{V}_{\\pi}(s) = \\sum_{a \\in \\mathcal{A}} \\pi(a | s) \\mathcal{Q}(s, a)\\) Loose intuitive interpretation : state-value = sum(policy determining actions * respective action-values) Basic: Action-value function \\(\\mathcal{Q}_{\\pi}(s, a)\\) With a list of possible multiple actions, there is a list of possible subsequent states \\(s'\\) associated with: state value function \\(\\mathcal{V}_{\\pi}(s')\\) transition probability function \\(\\mathcal{P}_{ss'}^a\\) determining where the agent could land in based on the action reward \\(\\mathcal{R}_s^a\\) for taking the action Summing the reward and the transition probability function associated with the state-value function gives us an indication of how good it is to take the actions given our state Final equation: \\(\\mathcal{Q}_{\\pi}(s, a) = \\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a {V}_{\\pi}(s')\\) Loose intuitive interpretation : action-value = reward + sum(transition outcomes determining states * respective state-values) Expanded functions (substitution) Substituting action-value function into the state-value function \\(\\mathcal{V}_{\\pi}(s) = \\sum_{a \\in \\mathcal{A}} \\pi(a | s) (\\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a {V}_{\\pi}(s'))\\) Substituting state-value function into action-value function \\(\\mathcal{Q}_{\\pi}(s, a) = \\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a \\sum_{a' \\in \\mathcal{A}} \\pi(a' | s') \\mathcal{Q}(s', a')\\) Bellman Optimality Equations \u00b6 Remember optimal policy \\(\\pi_*\\) \u2192 optimal state-value and action-value functions \u2192 argmax of value functions \\(\\pi_{*} = \\arg\\max_{\\pi} \\mathcal{V}_{\\pi}(s) = \\arg\\max_{\\pi} \\mathcal{Q}_{\\pi}(s, a)\\) Finally with Bellman Expectation Equations derived from Bellman Equations, we can derive the equations for the argmax of our value functions Optimal state-value function \\(\\mathcal{V}_*(s) = \\arg\\max_{\\pi} \\mathcal{V}_{\\pi}(s)\\) Given \\(\\mathcal{V}_{\\pi}(s) = \\sum_{a \\in \\mathcal{A}} \\pi(a | s) (\\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a {V}_{\\pi}(s'))\\) We have \\(\\mathcal{V}_*(s) = \\max_{a \\in \\mathcal{A}} (\\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a {V}_{*}(s')))\\) Optimal action-value function \\(\\mathcal{Q}_*(s) = \\arg\\max_{\\pi} \\mathcal{Q}_{\\pi}(s)\\) Given \\(\\mathcal{Q}_{\\pi}(s, a) = \\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a \\sum_{a' \\in \\mathcal{A}} \\pi(a' | s') \\mathcal{Q}(s', a')\\) We have \\(\\mathcal{Q}_{*}(s, a) = \\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a max_{a' \\in \\mathcal{A}} \\mathcal{Q}_{*}(s', a')\\) Optimal Action-value and State-value functions \u00b6 If the entire environment is known, such that we know our reward function and transition probability function, then we can solve for the optimal action-value and state-value functions via Dynamic Programming like Policy evaluation, policy improvement, and policy iteration However, typically we don't know the environment entirely then there is not closed form solution in getting optimal action-value and state-value functions. Hence, we need other iterative approaches like Monte-Carlo methods Temporal difference learning (model-free and learns with episodes) On-policy TD: SARSA Off-policy TD: Q-Learning and Deep Q-Learning (DQN) Policy gradient REINFORCE Actor-Critic A2C/A3C ACKTR PPO DPG DDPG (DQN + DPG) Closed form solution If there is a closed form solution, then the variables' values can be obtained with a finite number of mathematical operations (for example add, subtract, divide, and multiply). For example, solving \\(2x = 8 - 6x\\) would yield \\(8x = 8\\) by adding \\(6x\\) on both sides of the equation and finally yielding the value of \\(x=1\\) by dividing both sides of the equation by \\(8\\) . These finite 2 steps of mathematical operations allowed us to solve for the value of x as the equation has a closed-form solution. However, many cases in deep learning and reinforcement learning there are no closed-form solutions which requires all the iterative methods mentioned above. Bellman, R. A Markovian Decision Process. Journal of Mathematics and Mechanics. 1957. \u21a9 Matthew J. Hausknecht and Peter Stone. Deep Recurrent Q-Learning for Partially Observable MDPs . 2015. \u21a9 R Bellman. On the Theory of Dynamic Programming. Proceedings of the National Academy of Sciences. 1952. \u21a9","title":"Markov Decision Processes (MDP) and Bellman Equations"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#markov-decision-processes-mdps","text":"Typically we can frame all RL tasks as MDPs 1 Intuitively, it's sort of a way to frame RL tasks such that we can solve them in a \"principled\" manner. We will go into the specifics throughout this tutorial The key in MDPs is the Markov Property Essentially the future depends on the present and not the past More specifically, the future is independent of the past given the present There's an assumption the present state encapsulates past information. This is not always true, see the note below. Putting into the context of what we have covered so far: our agent can (1) control its action based on its current (2) completely known state Back to the \"driving to avoid puppy\" example: given we know there is a dog in front of the car as the current state and the car is always moving forward (no reverse driving), the agent can decide to take a left/right turn to avoid colliding with the puppy in front","title":"Markov Decision Processes (MDPs)"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#two-main-characteristics-for-mdps","text":"Control over state transitions States completely observable Other Markov Models Permutations of whether there is presence of the two main characteristics would lead to different Markov models. These are not important now, but it gives you an idea of what other frameworks we can use besides MDPs.","title":"Two main characteristics for MDPs"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#types-of-markov-models","text":"{++Control++} over state transitions and {++completely observable++} states: MDPs {++Control++} over state transitions and partially observable states : Partially Observable MDPs (POMDPs) No control over state transitions and {++completely observable++} states: Markov Chain No control over state transitions and partially observable states: Hidden Markov Model","title":"Types of Markov Models"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#pomdps","text":"Imagine our driving example where we don't know if the car is going forward/backward in its state, but only know there is a puppy in the center lane in front, this is a partially observable state There are ways to counter this Use the complete history to construct the current state Represent the current state as a probability distribution ( bayesian approach ) of what the agent perceives of the current state Using a RNN to form the current state that encapsulates the past 2","title":"POMDPs"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#5-components-of-mdps","text":"\\(\\mathcal{S}\\) : set of states \\(\\mathcal{A}\\) : set of actions \\(\\mathcal{R}\\) : reward function \\(\\mathcal{P}\\) : transition probability function \\(\\gamma\\) : discount for future rewards Remembering 5 components with a mnemonic A mnemonic I use to remember the 5 components is the acronym \"SARPY\" (sar-py). I know \\(\\gamma\\) is not \\(\\mathcal{Y}\\) but it looks like a y so there's that.","title":"5 Components of MDPs"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#moving-from-mdps-to-optimal-policy","text":"We have an agent acting in an environment The way the environment reacts to the agent's actions ( \\(a\\) ) is dictated by a model The agent can take actions ( \\(a\\) ) to move from one state ( \\(s\\) ) to another new state ( \\(s')\\) When the agent has transited to a new state ( \\(s')\\) , there will a reward ( \\(r\\) ) We may or may not know our model Model-based RL : this is where we can {++clearly define++} our (1) transition probabilities and/or (2) reward function A global minima can be attained via Dynamic Programming (DP) Model-free RL : this is where we cannot clearly define our (1) transition probabilities and/or (2) reward function Most real-world problems are under this category so we will mostly place our attention on this category How the agent acts ( \\(a\\) ) in its current state ( \\(s\\) ) is specified by its policy ( \\(\\pi(s)\\) ) It can either be deterministic or stochastic Deterministic policy : \\(a = \\pi(s)\\) Stochastic policy : \\(\\mathbb{P}_\\pi [A=a \\vert S=s] = \\pi(a | s)\\) This is the proability of taking an action given the current state under the policy \\(\\mathcal{A}\\) : set of all actions \\(\\mathcal{S}\\) : set of all states When the agent acts given its state under the policy ( \\(\\pi(a | s)\\) ) , the transition probability function \\(\\mathcal{P}\\) determines the subsequent state ( \\(s'\\) ) \\(\\mathcal{P}_{ss'}^a = \\mathcal{P}(s' \\vert s, a) = \\mathbb{P} [S_{t+1} = s' \\vert S_t = s, A_t = a]\\) When the agent act based on its policy ( \\(\\pi(a | s)\\) ) and transited to the new state determined by the transition probability function \\(\\mathcal{P}_{ss'}^a\\) it gets a reward based on the reward function as a feedback \\(\\mathcal{R}_s^a = \\mathbb{E} [\\mathcal{R}_{t+1} \\vert S_t = s, A_t = a]\\) Rewards are short-term, given as feedback after the agent takes an action and transits to a new state. Summing all future rewards and discounting them would lead to our return \\(\\mathcal{G}\\) \\(\\mathcal{G}_t = \\sum_{i=0}^{N} \\gamma^k \\mathcal{R}_{t+1+i}\\) \\(\\gamma\\) , our discount factor which ranges from 0 to 1 (inclusive) reduces the weightage of future rewards allowing us to balance between short-term and long-term goals With our return \\(\\mathcal{G}\\) , we then have our state-value function \\(\\mathcal{V}_{\\pi}\\) (how good to stay in that state) and our action-value or q-value function \\(\\mathcal{Q}_{\\pi}\\) (how good to take the action) \\(\\mathcal{V}_{\\pi}(s) = \\mathbb{E}_{\\pi}[\\mathcal{G}_t \\vert \\mathcal{S}_t = s]\\) \\(\\mathcal{Q}_{\\pi}(s, a) = \\mathbb{E}_{\\pi}[\\mathcal{G}_t \\vert \\mathcal{S}_t = s, \\mathcal{A}_t = a]\\) The advantage function is simply the difference between the two functions \\(\\mathcal{A}_{\\pi}(s, a) = \\mathcal{Q}_{\\pi}(s, a) - \\mathcal{V}_{\\pi}(s)\\) Seems useless at this stage, but this advantage function will be used in some key algorithms we are covering Since our policy determines how our agent acts given its state, achieving an optimal policy \\(\\pi_*\\) would mean achieving optimal actions that is exactly what we want! Basic Categories of Approaches We've covered state-value functions, action-value functions, model-free RL and model-based RL. They form general overarching categories of how we design our agent.","title":"Moving From MDPs to Optimal Policy"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#categories-of-design","text":"State-value based: search for the optimal state-value function (goodness of action in the state) Action-value based: search for the optimal action-value function (goodness of policy) Actor-critic based: using both state-value and action-value function Model based: attempts to model the environment to find the best policy Model-free based: trial and error to optimize for the best policy to get the most rewards instead of modelling the environment explicitly","title":"Categories of Design"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#optimal-policy","text":"Optimal policy \\(\\pi_*\\) \u2192 optimal state-value and action-value functions \u2192 max return \u2192 argmax of value functions \\(\\pi_{*} = \\arg\\max_{\\pi} \\mathcal{V}_{\\pi}(s) = \\arg\\max_{\\pi} \\mathcal{Q}_{\\pi}(s, a)\\) To calculate argmax of value functions \u2192 we need max return \\(\\mathcal{G}_t\\) \u2192 need max sum of rewards \\(\\mathcal{R}_s^a\\) To get max sum of rewards \\(\\mathcal{R}_s^a\\) we will rely on the Bellman Equations. 3","title":"Optimal Policy"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#bellman-equation","text":"Essentially, the Bellman Equation breaks down our value functions into two parts Immediate reward Discounted future value function State-value function can be broken into: \\(\\begin{aligned} \\mathcal{V}_{\\pi}(s) &= \\mathbb{E}[\\mathcal{G}_t \\vert \\mathcal{S}_t = s] \\\\ &= \\mathbb{E} [\\mathcal{R}_{t+1} + \\gamma \\mathcal{R}_{t+2} + \\gamma^2 \\mathcal{R}_{t+3} + \\dots \\vert \\mathcal{S}_t = s] \\\\ &= \\mathbb{E} [\\mathcal{R}_{t+1} + \\gamma (\\mathcal{R}_{t+2} + \\gamma \\mathcal{R}_{t+3} + \\dots) \\vert \\mathcal{S}_t = s] \\\\ &= \\mathbb{E} [\\mathcal{R}_{t+1} + \\gamma \\mathcal{G}_{t+1} \\vert \\mathcal{S}_t = s] \\\\ &= \\mathbb{E} [\\mathcal{R}_{t+1} + \\gamma \\mathcal{V}_{\\pi}(\\mathcal{s}_{t+1}) \\vert \\mathcal{S}_t = s] \\end{aligned}\\) Action-value function can be broken into: \\(\\mathcal{Q}_{\\pi}(s, a) = \\mathbb{E} [\\mathcal{R}_{t+1} + \\gamma \\mathcal{Q}_{\\pi}(\\mathcal{s}_{t+1}, \\mathcal{a}_{t+1}) \\vert \\mathcal{S}_t = s, \\mathcal{A} = a]\\)","title":"Bellman Equation"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#key-recap-on-value-functions","text":"\\(\\mathcal{V}_{\\pi}(s) = \\mathbb{E}_{\\pi}[\\mathcal{G}_t \\vert \\mathcal{S}_t = s]\\) State-value function: tells us how good to be in that state \\(\\mathcal{Q}_{\\pi}(s, a) = \\mathbb{E}_{\\pi}[\\mathcal{G}_t \\vert \\mathcal{S}_t = s, \\mathcal{A}_t = a]\\) Action-value function: tells us how good to take actions given state","title":"Key Recap on Value Functions"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#bellman-expectation-equations","text":"Now we can move from Bellman Equations into Bellman Expectation Equations Basic: State-value function \\(\\mathcal{V}_{\\pi}(s)\\) Current state \\(\\mathcal{S}\\) Multiple possible actions determined by stochastic policy \\(\\pi(a | s)\\) Each possible action is associated with a action-value function \\(\\mathcal{Q}_{\\pi}(s, a)\\) returning a value of that particular action Multiplying the possible actions with the action-value function and summing them gives us an indication of how good it is to be in that state Final equation: \\(\\mathcal{V}_{\\pi}(s) = \\sum_{a \\in \\mathcal{A}} \\pi(a | s) \\mathcal{Q}(s, a)\\) Loose intuitive interpretation : state-value = sum(policy determining actions * respective action-values) Basic: Action-value function \\(\\mathcal{Q}_{\\pi}(s, a)\\) With a list of possible multiple actions, there is a list of possible subsequent states \\(s'\\) associated with: state value function \\(\\mathcal{V}_{\\pi}(s')\\) transition probability function \\(\\mathcal{P}_{ss'}^a\\) determining where the agent could land in based on the action reward \\(\\mathcal{R}_s^a\\) for taking the action Summing the reward and the transition probability function associated with the state-value function gives us an indication of how good it is to take the actions given our state Final equation: \\(\\mathcal{Q}_{\\pi}(s, a) = \\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a {V}_{\\pi}(s')\\) Loose intuitive interpretation : action-value = reward + sum(transition outcomes determining states * respective state-values) Expanded functions (substitution) Substituting action-value function into the state-value function \\(\\mathcal{V}_{\\pi}(s) = \\sum_{a \\in \\mathcal{A}} \\pi(a | s) (\\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a {V}_{\\pi}(s'))\\) Substituting state-value function into action-value function \\(\\mathcal{Q}_{\\pi}(s, a) = \\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a \\sum_{a' \\in \\mathcal{A}} \\pi(a' | s') \\mathcal{Q}(s', a')\\)","title":"Bellman Expectation Equations"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#bellman-optimality-equations","text":"Remember optimal policy \\(\\pi_*\\) \u2192 optimal state-value and action-value functions \u2192 argmax of value functions \\(\\pi_{*} = \\arg\\max_{\\pi} \\mathcal{V}_{\\pi}(s) = \\arg\\max_{\\pi} \\mathcal{Q}_{\\pi}(s, a)\\) Finally with Bellman Expectation Equations derived from Bellman Equations, we can derive the equations for the argmax of our value functions Optimal state-value function \\(\\mathcal{V}_*(s) = \\arg\\max_{\\pi} \\mathcal{V}_{\\pi}(s)\\) Given \\(\\mathcal{V}_{\\pi}(s) = \\sum_{a \\in \\mathcal{A}} \\pi(a | s) (\\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a {V}_{\\pi}(s'))\\) We have \\(\\mathcal{V}_*(s) = \\max_{a \\in \\mathcal{A}} (\\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a {V}_{*}(s')))\\) Optimal action-value function \\(\\mathcal{Q}_*(s) = \\arg\\max_{\\pi} \\mathcal{Q}_{\\pi}(s)\\) Given \\(\\mathcal{Q}_{\\pi}(s, a) = \\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a \\sum_{a' \\in \\mathcal{A}} \\pi(a' | s') \\mathcal{Q}(s', a')\\) We have \\(\\mathcal{Q}_{*}(s, a) = \\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a max_{a' \\in \\mathcal{A}} \\mathcal{Q}_{*}(s', a')\\)","title":"Bellman Optimality Equations"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#optimal-action-value-and-state-value-functions","text":"If the entire environment is known, such that we know our reward function and transition probability function, then we can solve for the optimal action-value and state-value functions via Dynamic Programming like Policy evaluation, policy improvement, and policy iteration However, typically we don't know the environment entirely then there is not closed form solution in getting optimal action-value and state-value functions. Hence, we need other iterative approaches like Monte-Carlo methods Temporal difference learning (model-free and learns with episodes) On-policy TD: SARSA Off-policy TD: Q-Learning and Deep Q-Learning (DQN) Policy gradient REINFORCE Actor-Critic A2C/A3C ACKTR PPO DPG DDPG (DQN + DPG) Closed form solution If there is a closed form solution, then the variables' values can be obtained with a finite number of mathematical operations (for example add, subtract, divide, and multiply). For example, solving \\(2x = 8 - 6x\\) would yield \\(8x = 8\\) by adding \\(6x\\) on both sides of the equation and finally yielding the value of \\(x=1\\) by dividing both sides of the equation by \\(8\\) . These finite 2 steps of mathematical operations allowed us to solve for the value of x as the equation has a closed-form solution. However, many cases in deep learning and reinforcement learning there are no closed-form solutions which requires all the iterative methods mentioned above. Bellman, R. A Markovian Decision Process. Journal of Mathematics and Mechanics. 1957. \u21a9 Matthew J. Hausknecht and Peter Stone. Deep Recurrent Q-Learning for Partially Observable MDPs . 2015. \u21a9 R Bellman. On the Theory of Dynamic Programming. Proceedings of the National Academy of Sciences. 1952. \u21a9","title":"Optimal Action-value and State-value functions"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/","text":"Frozen Lake Introduction \u00b6 Run Jupyter Notebook You can run the code for this section in this jupyter notebook link . Fronze Lake is a simple game where you are on a frozen lake and you need to retrieve an item on the frozen lake where some parts are frozen and some parts are holes (if you walk into them you die) Actions: \\(\\mathcal{A} = \\{0, 1, 2, 3\\}\\) LEFT: 0 DOWN = 1 RIGHT = 2 UP = 3 Whole lake is a 4 x 4 grid world, \\(\\mathcal{S} = \\{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\\}\\) On each grid, there are 4 possibilities S: starting point, safe (code = 'SFFF') F: frozen surface, safe (code = 'FHFH') H: hole, fall to your doom (code = 'FFFH') G: goal, where the frisbee is located ('HFFG') Goal of Frozen Lake \u00b6 The key here is we want to get to G without falling into the hole H in the shortest amount of time Why Dynamic Programming? \u00b6 In this game, we know our transition probability function and reward function, essentially the whole environment, allowing us to turn this game into a simple planning problem via dynamic programming through 4 simple functions: (1) policy evaluation (2) policy improvement (3) policy iteration or (4) value iteration Before we explore how to solve this game, let's first understand how the game works in detail. Deterministic Policy Environment \u00b6 Make OpenAI Gym Environment for Frozen Lake # Import gym, installable via `pip install gym` import gym # Environment environment Slippery (stochastic policy, move left probability = 1/3) comes by default! # If we want deterministic policy, we need to create new environment # Make environment No Slippery (deterministic policy, move left = left) gym . envs . register ( id = 'FrozenLakeNotSlippery-v0' , entry_point = 'gym.envs.toy_text:FrozenLakeEnv' , kwargs = { 'map_name' : '4x4' , 'is_slippery' : False }, max_episode_steps = 100 , reward_threshold = 0.78 , # optimum = .8196 ) # You can only register once # To delete any new environment # del gym.envs.registry.env_specs['FrozenLakeNotSlippery-v0'] # Make the environment based on deterministic policy env = gym . make ( 'FrozenLakeNotSlippery-v0' ) Observation space # State space print ( env . observation_space ) Discrete(16) State space S_n = env . observation_space . n print ( S_n ) 16 Sampling state space # We should expect to see 15 possible grids from 0 to 15 when # we uniformly randomly sample from our observation space for i in range ( 10 ): print ( env . observation_space . sample ()) 11 12 7 13 7 11 14 14 4 12 Action space # Action space print ( env . action_space ) A_n = env . action_space . n print ( A_n ) Discrete(4) 4 Random sampling of actions # We should expect to see 4 actions when # we uniformly randomly sample: # 1. LEFT: 0 # 2. DOWN = 1 # 3. RIGHT = 2 # 4. UP = 3 for i in range ( 10 ): print ( env . action_space . sample ()) 0 3 1 3 3 2 3 2 2 2 Making Steps \u00b6 Initial state # This sets the initial state at S, our starting point # We can render the environment to see where we are on the 4x4 frozenlake gridworld env . reset () env . render () [S]FFF FHFH FFFH HFFG Go left # Go left (action=0), nothing should happen, and we should stay at the starting point, because there's no grid on the left env . reset () action = 0 ( observation , reward , done , prob ) = env . step ( action ) env . render () # Observation = 1: move to grid number 1 (unchanged) # Prob = 1: deterministic policy, if we choose to go left, we'll go left print ( observation , reward , done , prob ) (Left) [S]FFF FHFH FFFH HFFG 0 0.0 False {'prob': 1.0} Go down # Go down (action = 1), we should be safe as we step on frozen grid env . reset () action = 1 ( observation , reward , done , prob ) = env . step ( action ) env . render () # Observation = 4: move to grid number 4 # Prob = 1: deterministic policy, if we choose to go down we'll go down print ( observation , reward , done , prob ) (Down) SFFF [F]HFH FFFH HFFG 4 0.0 False {'prob': 1.0} Go right # Go right (action = 2), we should be safe as we step on frozen grid env . reset () action = 2 ( observation , reward , done , prob ) = env . step ( action ) env . render () # Observation = 1: move to grid number 1 # Prob = 1: deterministic policy, if we choose to go right we'll go right print ( observation , reward , done , prob ) (Right) S[F]FF FHFH FFFH HFFG 1 0.0 False {'prob': 1.0} Go right twice # Go right twice (action = 2), we should be safe as we step on 2 frozen grids env . reset () action = 2 ( observation , reward , done , prob ) = env . step ( action ) env . render () ( observation , reward , done , prob ) = env . step ( action ) env . render () # Observation = 2: move to the right twice from grid 0 to grid 2 # Prob = 1: deterministic policy, if we choose to go right twice we'll go right twice print ( observation , reward , done , prob ) (Right) S[F]FF FHFH FFFH HFFG (Right) SF[F]F FHFH FFFH HFFG 2 0.0 False {'prob': 1.0} Dying: drop in hole grid 12, H \u00b6 Go down thrice # Go down thrice (action = 1), we will die as we step onto the grid with a hole env . reset () action = 1 ( observation , reward , done , prob ) = env . step ( action ) env . render () ( observation , reward , done , prob ) = env . step ( action ) env . render () ( observation , reward , done , prob ) = env . step ( action ) env . render () # Observation = 2: move to the right twice from grid 0 to grid 2 # Prob = 1: deterministic policy, if we choose to go right twice we'll go right twice # Done = True because the game ends when we die (go onto hole grid (H) or finish the game (G)) print ( observation , reward , done , prob ) (Down) SFFF [F]HFH FFFH HFFG (Down) SFFF FHFH [F]FFH HFFG (Down) SFFF FHFH FFFH [H]FFG 12 0.0 True {'prob': 1.0} Winning: get to grid 15, G \u00b6 Go right twice, go down thrice, go right once # Go right twice (action = 2), go down thrice (action = 1), go right once (action = 2) env . reset () # Right Twice action = 2 ( observation , reward , done , prob ) = env . step ( action ) env . render () ( observation , reward , done , prob ) = env . step ( action ) env . render () # Down Thrice action = 1 ( observation , reward , done , prob ) = env . step ( action ) env . render () ( observation , reward , done , prob ) = env . step ( action ) env . render () ( observation , reward , done , prob ) = env . step ( action ) env . render () # Right Once action = 2 ( observation , reward , done , prob ) = env . step ( action ) env . render () # Observation = 2: move to the right twice from grid 0 to grid 2 # Prob = 1: deterministic policy, if we choose to go right twice we'll go right twice # Done = True because the game ends when we die (go onto hole grid (H) or finish the game (G)) print ( observation , reward , done , prob ) (Right) S[F]FF FHFH FFFH HFFG (Right) SF[F]F FHFH FFFH HFFG (Down) SFFF FH[F]H FFFH HFFG (Down) SFFF FHFH FF[F]H HFFG (Down) SFFF FHFH FFFH HF[F]mG (Right) SFFF FHFH FFFH HFF[G] 15 1.0 True {'prob': 1.0} Non-deterministic Policy Environment \u00b6 Go right? # Make the environment based on non-deterministic policy env = gym . make ( 'FrozenLake-v0' ) # Go right once (action = 2), we should go to the right but we did not! env . seed ( 8 ) env . reset () action = 2 ( observation , reward , done , prob ) = env . step ( action ) env . render () # Observation = 0: move to the right once from grid 0 to grid 1 # Prob = 1/3: non-deterministic policy, if we choose to go right, there's only a 1/3 probability we would go to the right and with this environment seed we did not print ( observation , reward , done , prob ) (Right) [S]FFF FHFH FFFH HFFG 0 0.0 False {'prob': 0.3333333333333333} Go right 10 times? # Try to go to the right 10 times, let's see how many times it goes to the right, by right we won't die because we would end up at the extreme right of grid 3 # See how it can go down/left/up/nothing instead of just right? # Intuitively when we are moving on a frozen lake, some times when we want to walk one direction we may end up in another direction as it's slippery # Setting seed here of the environment so you can reproduce my results, otherwise stochastic policy will yield different results for each run env . seed ( 8 ) env . reset () for i in range ( 10 ): action = 2 ( observation , reward , done , prob ) = env . step ( action ) env . render () (Right) [S]FFF FHFH FFFH HFFG (Right) S[F]FF FHFH FFFH HFFG (Right) SF[F]F FHFH FFFH HFFG (Right) SFFF FH[F]H FFFH HFFG (Right) SFFF FHF[H] FFFH HFFG (Right) SFFF FHF[4H] FFFH HFFG (Right) SFFF FHF[H] FFFH HFFG (Right) SFFF FHF[H] FFFH HFFG (Right) SFFF FHF[H] FFFH HFFG (Right) SFFF FHF[H] FFFH HFFG Custom Frozen Lake Non-deterministic Policy Environment \u00b6 Because original code from OpenAI only allows us to run env.step(action) , this is challenging if we want to do some visualization of our state-value and action-value (q-value) functions for learning Hence, we'll be copying the whole code from OpenAI Frozen Lake implementation and adding just one line to make sure we can get P via self.P = P This code is not important, you can just copy it import sys from contextlib import closing import numpy as np from six import StringIO , b from gym import utils from gym.envs.toy_text import discrete LEFT = 0 DOWN = 1 RIGHT = 2 UP = 3 MAPS = { \"4x4\" : [ \"SFFF\" , \"FHFH\" , \"FFFH\" , \"HFFG\" ], \"8x8\" : [ \"SFFFFFFF\" , \"FFFFFFFF\" , \"FFFHFFFF\" , \"FFFFFHFF\" , \"FFFHFFFF\" , \"FHHFFFHF\" , \"FHFFHFHF\" , \"FFFHFFFG\" ], } # Generates a random valid map (one that has a path from start to goal) # @params size, size of each side of the grid # @prams p, probability that a tile is frozen def generate_random_map ( size = 8 , p = 0.8 ): valid = False #BFS to check that it's a valid path def is_valid ( arr , r = 0 , c = 0 ): if arr [ r ][ c ] == 'G' : return True tmp = arr [ r ][ c ] arr [ r ][ c ] = \"#\" if r + 1 < size and arr [ r + 1 ][ c ] not in '#H' : if is_valid ( arr , r + 1 , c ) == True : arr [ r ][ c ] = tmp return True if c + 1 < size and arr [ r ][ c + 1 ] not in '#H' : if is_valid ( arr , r , c + 1 ) == True : arr [ r ][ c ] = tmp return True if r - 1 >= 0 and arr [ r - 1 ][ c ] not in '#H' : if is_valid ( arr , r - 1 , c ) == True : arr [ r ][ c ] = tmp return True if c - 1 >= 0 and arr [ r ][ c - 1 ] not in '#H' : if is_valid ( arr , r , c - 1 ) == True : arr [ r ][ c ] = tmp return True arr [ r ][ c ] = tmp return False while not valid : p = min ( 1 , p ) res = np . random . choice ([ 'F' , 'H' ], ( size , size ), p = [ p , 1 - p ]) res [ 0 ][ 0 ] = 'S' res [ - 1 ][ - 1 ] = 'G' valid = is_valid ( res ) return [ \"\" . join ( x ) for x in res ] class FrozenLakeEnv ( discrete . DiscreteEnv ): \"\"\" Winter is here. You and your friends were tossing around a frisbee at the park when you made a wild throw that left the frisbee out in the middle of the lake. The water is mostly frozen, but there are a few holes where the ice has melted. If you step into one of those holes, you'll fall into the freezing water. At this time, there's an international frisbee shortage, so it's absolutely imperative that you navigate across the lake and retrieve the disc. However, the ice is slippery, so you won't always move in the direction you intend. The surface is described using a grid like the following SFFF FHFH FFFH HFFG S : starting point, safe F : frozen surface, safe H : hole, fall to your doom G : goal, where the frisbee is located The episode ends when you reach the goal or fall in a hole. You receive a reward of 1 if you reach the goal, and zero otherwise. \"\"\" metadata = { 'render.modes' : [ 'human' , 'ansi' ]} def __init__ ( self , desc = None , map_name = \"4x4\" , is_slippery = True ): if desc is None and map_name is None : desc = generate_random_map () elif desc is None : desc = MAPS [ map_name ] self . desc = desc = np . asarray ( desc , dtype = 'c' ) self . nrow , self . ncol = nrow , ncol = desc . shape self . reward_range = ( 0 , 1 ) nA = 4 nS = nrow * ncol isd = np . array ( desc == b 'S' ) . astype ( 'float64' ) . ravel () isd /= isd . sum () P = { s : { a : [] for a in range ( nA )} for s in range ( nS )} def to_s ( row , col ): return row * ncol + col def inc ( row , col , a ): if a == 0 : # left col = max ( col - 1 , 0 ) elif a == 1 : # down row = min ( row + 1 , nrow - 1 ) elif a == 2 : # right col = min ( col + 1 , ncol - 1 ) elif a == 3 : # up row = max ( row - 1 , 0 ) return ( row , col ) for row in range ( nrow ): for col in range ( ncol ): s = to_s ( row , col ) for a in range ( 4 ): li = P [ s ][ a ] letter = desc [ row , col ] if letter in b 'GH' : li . append (( 1.0 , s , 0 , True )) else : if is_slippery : for b in [( a - 1 ) % 4 , a , ( a + 1 ) % 4 ]: newrow , newcol = inc ( row , col , b ) newstate = to_s ( newrow , newcol ) newletter = desc [ newrow , newcol ] done = bytes ( newletter ) in b 'GH' rew = float ( newletter == b 'G' ) li . append (( 1.0 / 3.0 , newstate , rew , done )) else : newrow , newcol = inc ( row , col , a ) newstate = to_s ( newrow , newcol ) newletter = desc [ newrow , newcol ] done = bytes ( newletter ) in b 'GH' rew = float ( newletter == b 'G' ) li . append (( 1.0 , newstate , rew , done )) # New change because environment only allows step without # specific state for learning environment! self . P = P super ( FrozenLakeEnv , self ) . __init__ ( nS , nA , P , isd ) def render ( self , mode = 'human' ): outfile = StringIO () if mode == 'ansi' else sys . stdout row , col = self . s // self . ncol , self . s % self . ncol desc = self . desc . tolist () desc = [[ c . decode ( 'utf-8' ) for c in line ] for line in desc ] desc [ row ][ col ] = utils . colorize ( desc [ row ][ col ], \"red\" , highlight = True ) if self . lastaction is not None : outfile . write ( \" ( {} ) \\n \" . format ([ \"Left\" , \"Down\" , \"Right\" , \"Up\" ][ self . lastaction ])) else : outfile . write ( \" \\n \" ) outfile . write ( \" \\n \" . join ( '' . join ( line ) for line in desc ) + \" \\n \" ) if mode != 'human' : with closing ( outfile ): return outfile . getvalue () Policy Evaluation \u00b6 Transition Probability Function \u00b6 \\(\\mathcal{P}_{ss'}^a = \\mathcal{P}(s' \\vert s, a) = \\mathbb{P} [S_{t+1} = s' \\vert S_t = s, A_t = a]\\) Deterministic Environment \u00b6 There's no probability distribution, if you decide to go left you'll go left Hence in this example, given current_state = 8 and action = 0 which is left, we will end up with probability = 1 in new_state = 9 # Deterministic env = FrozenLakeEnv ( is_slippery = False ) current_state = 10 # State from S_n=16 State space action = 0 # Left action from A_n=4 Action space [( probability , new_state , reward , done )] = env . P [ current_state ][ action ] print ( 'Probability {} , New State {} ' . format ( probability , new_state )) Probability 1.0, New State 9 Stochastic Environment \u00b6 Given \\(S_t = 10, A_t = 0\\) in a stochastic environment, the transition probability functions indicate you can end up in grid 6, 9, 14 each with \u2153 probability: \\(\\mathbb{P} [S_{t+1} = 6 \\vert S_t = 10, A_t = 0] = \\frac{1}{3}\\) \\(\\mathbb{P} [S_{t+1} = 9 \\vert S_t = 10, A_t = 0] = \\frac{1}{3}\\) \\(\\mathbb{P} [S_{t+1} = 14 \\vert S_t = 10, A_t = 0] = \\frac{1}{3}\\) # Stochastic env = FrozenLakeEnv ( is_slippery = True ) current_state = 10 # State from S_n=16 State space action = 0 # Left action from A_n=4 Action space env . P [ current_state ][ action ] [(0.3333333333333333, 6, 0.0, False), (0.3333333333333333, 9, 0.0, False), (0.3333333333333333, 14, 0.0, False)] Random Policy Function \u00b6 Random Policy function # Random policy generation def generate_random_policy ( S_n , A_n ): # return np.random.randint(A_n, size=(S_n, A_n)) return np . ones ([ S_n , A_n ]) / A_n # Given the total number of states S_n = 16 # For each state out of 16 states, we can take 4 actions # Since this is a stochastic environment, we'll initialize a policy to have equal probabilities 0.25 of doing each action each state policy = generate_random_policy ( S_n , A_n ) print ( policy . shape ) (16, 4) Policy plot import seaborn as sns import matplotlib.pyplot as plt % matplotlib inline plt . figure ( figsize = ( 5 , 16 )) sns . heatmap ( policy , cmap = \"YlGnBu\" , annot = True , cbar = False ); Policy Evaluation Function comprising State-value Function \u00b6 How: \\(\\mathcal{V}_{\\pi}(s) = \\sum_{a \\in \\mathcal{A}} \\pi(a | s) \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a \\big[\\mathcal{R}_s^a + \\gamma {V}_{\\pi}(s')\\big]\\) Simple code equation: Values of state given policy = sum ( action probability * transition probability * [reward + discount * value of new state] ) Aim: getting state-values import numpy as np def policy_evaluation ( env , policy , gamma = 1. , theta = 1e-8 ): r \"\"\"Policy evaluation function. Loop until state values stable, delta < theta. Returns V comprising values of states under given policy. Args: env (gym.env): OpenAI environment class instantiated and assigned to an object. policy (np.array): policy array to evaluate gamma (float): discount rate for rewards theta (float): tiny positive number, anything below it indicates value function convergence \"\"\" # 1. Create state-value array (16,) V = np . zeros ( S_n ) while True : delta = 0 # 2. Loop through states for s in range ( S_n ): Vs = 0 # 2.1 Loop through actions for the unique state # Given each state, we've 4 actions associated with different probabilities # 0.25 x 4 in this case, so we'll be looping 4 times (4 action probabilities) at each state for a , action_prob in enumerate ( policy [ s ]): # 2.1.1 Loop through to get transition probabilities, next state, rewards and whether the game ended for prob , next_state , reward , done in env . P [ s ][ a ]: # State-value function to get our values of states given policy Vs += action_prob * prob * ( reward + gamma * V [ next_state ]) # This simple equation allows us to stop this loop when we've converged # How do we know? The new value of the state is smaller than a tiny positive value we set # State value change is tiny compared to what we have so we just stop! delta = max ( delta , np . abs ( V [ s ] - Vs )) # 2.2 Update our state value for that state V [ s ] = Vs # 3. Stop policy evaluation if our state values changes are smaller than our tiny positive number if delta < theta : break return V # Generate random policy with equal probabilities of each action given any state rand_policy = generate_random_policy ( S_n , A_n ) # Evaluate the policy to get state values V = policy_evaluation ( env , rand_policy ) # Plot heatmap plt . figure ( figsize = ( 8 , 8 )) sns . heatmap ( V . reshape ( 4 , 4 ), cmap = \"YlGnBu\" , annot = True , cbar = False ); # This is our environment # Notice how the state values near the goal have higher values? # Those with \"H\" = hole, where you die if you step, have 0 values indicating those are bad areas to be in env . render () [S]FFF FHFH FFFH HFFG Policy Improvement \u00b6 Action-value (Q-value) function from State-value function \u00b6 How: \\(\\mathcal{Q}_{\\pi}(s, a) = \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a \\big[ \\mathcal{R}_s^a + \\gamma \\mathcal{V}_{\\pi}(s') \\big]\\) Code equation Values of action = sum ( transition probability * [reward + discount * value of next state] ) Aim: getting q-values (action-values) def q_value ( env , V , s , gamma = 1 ): r \"\"\"Q-value (action-value) function from state-value function Returns Q values, values of actions. Args: env (gym.env): OpenAI environment class instantiated and assigned to an object. V (np.array): array of state-values obtained from policy evaluation function. s (integer): integer representing current state in the gridworld gamma (float): discount rate for rewards. \"\"\" # 1. Create q-value array for one state # We have 4 actions, so let's create an array with the size of 4 q = np . zeros ( A_n ) # 2. Loop through each action for a in range ( A_n ): # 2.1 For each action, we've our transition probabilities, next state, rewards and whether the game ended for prob , next_state , reward , done in env . P [ s ][ a ]: # 2.1.1 Get our action-values from state-values q [ a ] += prob * ( reward + gamma * V [ next_state ]) # Return action values return q # For every state, we've 4 actions, hence we've 16 x 4 q values Q = np . zeros ([ S_n , A_n ]) # Loop through each state out of 16 # For each state, we will get the 4 q-values associated with the 4 actions for s in range ( env . nS ): Q [ s ] = q_value ( env , V , s ) plt . figure ( figsize = ( 5 , 16 )) sns . heatmap ( Q , cmap = \"YlGnBu\" , annot = True , cbar = False ); # Notice how 13/14, those in the last row of the gridworld just before reaching the goal of finishing the game, their action values are large? env . render () [S]FFF FHFH FFFH HFFG Policy Improvement Function \u00b6 How: maximizing q-values per state by choosing actions with highest q-values Aim: get improved policy def policy_improvement ( env , V , gamma = 1. ): r \"\"\"Function to improve the policy by utilizing state values and action (q) values. Args: env (gym.env): OpenAI environment class instantiated and assigned to an objects V (np.array): array of state-values obtained from policy evaluation function gamma (float): discount of rewards \"\"\" # 1. Blank policy policy = np . zeros ([ env . nS , env . nA ]) / env . nA # 2. For each state in 16 states for s in range ( env . nS ): # 2.1 Get q values: q.shape returns (4,) q = q_value ( env , V , s , gamma ) # 2.2 Find best action based on max q-value # np.argwhere(q==np.max(q)) gives the position of largest q value # given array([0.00852356, 0.01163091, 0.0108613 , 0.01550788]), this would return array([[3]]) of shape (1, 1) # .flatten() reduces the shape to (1,) where we've array([3]) best_a = np . argwhere ( q == np . max ( q )) . flatten () # 2.3 One-hot encode best action and store into policy array's row for that state # In our case where the best action is array([3]), this would return # array([0., 0., 0., 1.]) where position 3 is the best action # Now we can store the best action into our policy policy [ s ] = np . sum ([ np . eye ( env . nA )[ i ] for i in best_a ], axis = 0 ) / len ( best_a ) return policy new_policy = policy_improvement ( env , V ) plt . figure ( figsize = ( 5 , 16 )) sns . heatmap ( new_policy , cmap = \"YlGnBu\" , annot = True , cbar = False ); # Compared to this equiprobable policy, the one above is making some improvements by maximizing q-values per state plt . figure ( figsize = ( 5 , 16 )) sns . heatmap ( rand_policy , cmap = \"YlGnBu\" , annot = True , cbar = False ); Policy Iteration Function \u00b6 How: loop through policy evaluation (get state-values) and policy improvement functions (use state-values to calculate q-values to improve policy) until optimal policy obtained Aim: improve policy until convergence Convergence: difference of state values between old and new policies is very small (less than theta, a very small positive number) import copy def policy_iteration ( env , gamma = 1 , theta = 1e-8 ): # 1. Create equiprobable policy where every state has 4 actions with equal probabilities as a starting policy policy = np . ones ([ env . nS , env . nA ]) / env . nA # 2. Loop through policy_evaluation and policy_improvement functions while True : # 2.1 Get state-values V = policy_evaluation ( env , policy , gamma , theta ) # 2.2 Get new policy by getting q-values and maximizing q-values per state to get best action per state new_policy = policy_improvement ( env , V ) # 2.3 Stop if the value function estimates for successive policies has converged if np . max ( abs ( policy_evaluation ( env , policy ) - policy_evaluation ( env , new_policy ))) < theta * 1e2 : break ; # 2.4 Replace policy with new policy policy = copy . copy ( new_policy ) return policy , V # obtain the optimal policy and optimal state-value function policy_pi , V_pi = policy_iteration ( env ) # Optimal policy (pi) # LEFT = 0, DOWN = 1, RIGHT = 2, UP = 3 plt . figure ( figsize = ( 5 , 16 )) sns . heatmap ( policy_pi , cmap = \"YlGnBu\" , annot = True , cbar = False , square = True ); # State values plt . figure ( figsize = ( 8 , 8 )) sns . heatmap ( V_pi . reshape ( 4 , 4 ), cmap = \"YlGnBu\" , annot = True , cbar = False , square = True ); # State values without policy improvement, just evaluation plt . figure ( figsize = ( 8 , 8 )) sns . heatmap ( V . reshape ( 4 , 4 ), cmap = \"YlGnBu\" , annot = True , cbar = False ); Value iteration \u00b6 Alternative to policy iteration How: loop through to find optimal value function then get one-off policy Aim: improve value function until convergence Convergence: until difference in new and old state values are small (smaller than theta, small positive number) def value_iteration ( env , gamma = 1 , theta = 1e-8 ): # 1. Create state values of shape (16,) V = np . zeros ( env . nS ) # 2. Loop through q-value function until convergence while True : delta = 0 # 2.1 Loop through each state for s in range ( env . nS ): # 2.2 Archive old state value v = V [ s ] # 2.3 New state value = max of q-value V [ s ] = max ( q_value ( env , V , s , gamma )) delta = max ( delta , abs ( V [ s ] - v )) # 2.2 If state value changes small, converged if delta < theta : break # 3. Extract one-off policy with optimal state values policy = policy_improvement ( env , V , gamma ) return policy , V policy_vi , V_vi = value_iteration ( env ) # Optimal policy # LEFT = 0, DOWN = 1, RIGHT = 2, UP = 3 plt . figure ( figsize = ( 5 , 16 )) sns . heatmap ( policy_vi , cmap = \"YlGnBu\" , annot = True , cbar = False , square = True ); # State values plt . figure ( figsize = ( 8 , 8 )) sns . heatmap ( V_vi . reshape ( 4 , 4 ), cmap = \"YlGnBu\" , annot = True , cbar = False , square = True );","title":"Dynamic Programming"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#frozen-lake-introduction","text":"Run Jupyter Notebook You can run the code for this section in this jupyter notebook link . Fronze Lake is a simple game where you are on a frozen lake and you need to retrieve an item on the frozen lake where some parts are frozen and some parts are holes (if you walk into them you die) Actions: \\(\\mathcal{A} = \\{0, 1, 2, 3\\}\\) LEFT: 0 DOWN = 1 RIGHT = 2 UP = 3 Whole lake is a 4 x 4 grid world, \\(\\mathcal{S} = \\{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\\}\\) On each grid, there are 4 possibilities S: starting point, safe (code = 'SFFF') F: frozen surface, safe (code = 'FHFH') H: hole, fall to your doom (code = 'FFFH') G: goal, where the frisbee is located ('HFFG')","title":"Frozen Lake Introduction"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#goal-of-frozen-lake","text":"The key here is we want to get to G without falling into the hole H in the shortest amount of time","title":"Goal of Frozen Lake"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#why-dynamic-programming","text":"In this game, we know our transition probability function and reward function, essentially the whole environment, allowing us to turn this game into a simple planning problem via dynamic programming through 4 simple functions: (1) policy evaluation (2) policy improvement (3) policy iteration or (4) value iteration Before we explore how to solve this game, let's first understand how the game works in detail.","title":"Why Dynamic Programming?"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#deterministic-policy-environment","text":"Make OpenAI Gym Environment for Frozen Lake # Import gym, installable via `pip install gym` import gym # Environment environment Slippery (stochastic policy, move left probability = 1/3) comes by default! # If we want deterministic policy, we need to create new environment # Make environment No Slippery (deterministic policy, move left = left) gym . envs . register ( id = 'FrozenLakeNotSlippery-v0' , entry_point = 'gym.envs.toy_text:FrozenLakeEnv' , kwargs = { 'map_name' : '4x4' , 'is_slippery' : False }, max_episode_steps = 100 , reward_threshold = 0.78 , # optimum = .8196 ) # You can only register once # To delete any new environment # del gym.envs.registry.env_specs['FrozenLakeNotSlippery-v0'] # Make the environment based on deterministic policy env = gym . make ( 'FrozenLakeNotSlippery-v0' ) Observation space # State space print ( env . observation_space ) Discrete(16) State space S_n = env . observation_space . n print ( S_n ) 16 Sampling state space # We should expect to see 15 possible grids from 0 to 15 when # we uniformly randomly sample from our observation space for i in range ( 10 ): print ( env . observation_space . sample ()) 11 12 7 13 7 11 14 14 4 12 Action space # Action space print ( env . action_space ) A_n = env . action_space . n print ( A_n ) Discrete(4) 4 Random sampling of actions # We should expect to see 4 actions when # we uniformly randomly sample: # 1. LEFT: 0 # 2. DOWN = 1 # 3. RIGHT = 2 # 4. UP = 3 for i in range ( 10 ): print ( env . action_space . sample ()) 0 3 1 3 3 2 3 2 2 2","title":"Deterministic Policy Environment"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#making-steps","text":"Initial state # This sets the initial state at S, our starting point # We can render the environment to see where we are on the 4x4 frozenlake gridworld env . reset () env . render () [S]FFF FHFH FFFH HFFG Go left # Go left (action=0), nothing should happen, and we should stay at the starting point, because there's no grid on the left env . reset () action = 0 ( observation , reward , done , prob ) = env . step ( action ) env . render () # Observation = 1: move to grid number 1 (unchanged) # Prob = 1: deterministic policy, if we choose to go left, we'll go left print ( observation , reward , done , prob ) (Left) [S]FFF FHFH FFFH HFFG 0 0.0 False {'prob': 1.0} Go down # Go down (action = 1), we should be safe as we step on frozen grid env . reset () action = 1 ( observation , reward , done , prob ) = env . step ( action ) env . render () # Observation = 4: move to grid number 4 # Prob = 1: deterministic policy, if we choose to go down we'll go down print ( observation , reward , done , prob ) (Down) SFFF [F]HFH FFFH HFFG 4 0.0 False {'prob': 1.0} Go right # Go right (action = 2), we should be safe as we step on frozen grid env . reset () action = 2 ( observation , reward , done , prob ) = env . step ( action ) env . render () # Observation = 1: move to grid number 1 # Prob = 1: deterministic policy, if we choose to go right we'll go right print ( observation , reward , done , prob ) (Right) S[F]FF FHFH FFFH HFFG 1 0.0 False {'prob': 1.0} Go right twice # Go right twice (action = 2), we should be safe as we step on 2 frozen grids env . reset () action = 2 ( observation , reward , done , prob ) = env . step ( action ) env . render () ( observation , reward , done , prob ) = env . step ( action ) env . render () # Observation = 2: move to the right twice from grid 0 to grid 2 # Prob = 1: deterministic policy, if we choose to go right twice we'll go right twice print ( observation , reward , done , prob ) (Right) S[F]FF FHFH FFFH HFFG (Right) SF[F]F FHFH FFFH HFFG 2 0.0 False {'prob': 1.0}","title":"Making Steps"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#dying-drop-in-hole-grid-12-h","text":"Go down thrice # Go down thrice (action = 1), we will die as we step onto the grid with a hole env . reset () action = 1 ( observation , reward , done , prob ) = env . step ( action ) env . render () ( observation , reward , done , prob ) = env . step ( action ) env . render () ( observation , reward , done , prob ) = env . step ( action ) env . render () # Observation = 2: move to the right twice from grid 0 to grid 2 # Prob = 1: deterministic policy, if we choose to go right twice we'll go right twice # Done = True because the game ends when we die (go onto hole grid (H) or finish the game (G)) print ( observation , reward , done , prob ) (Down) SFFF [F]HFH FFFH HFFG (Down) SFFF FHFH [F]FFH HFFG (Down) SFFF FHFH FFFH [H]FFG 12 0.0 True {'prob': 1.0}","title":"Dying: drop in hole grid 12,  H"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#winning-get-to-grid-15-g","text":"Go right twice, go down thrice, go right once # Go right twice (action = 2), go down thrice (action = 1), go right once (action = 2) env . reset () # Right Twice action = 2 ( observation , reward , done , prob ) = env . step ( action ) env . render () ( observation , reward , done , prob ) = env . step ( action ) env . render () # Down Thrice action = 1 ( observation , reward , done , prob ) = env . step ( action ) env . render () ( observation , reward , done , prob ) = env . step ( action ) env . render () ( observation , reward , done , prob ) = env . step ( action ) env . render () # Right Once action = 2 ( observation , reward , done , prob ) = env . step ( action ) env . render () # Observation = 2: move to the right twice from grid 0 to grid 2 # Prob = 1: deterministic policy, if we choose to go right twice we'll go right twice # Done = True because the game ends when we die (go onto hole grid (H) or finish the game (G)) print ( observation , reward , done , prob ) (Right) S[F]FF FHFH FFFH HFFG (Right) SF[F]F FHFH FFFH HFFG (Down) SFFF FH[F]H FFFH HFFG (Down) SFFF FHFH FF[F]H HFFG (Down) SFFF FHFH FFFH HF[F]mG (Right) SFFF FHFH FFFH HFF[G] 15 1.0 True {'prob': 1.0}","title":"Winning: get to grid 15, G"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#non-deterministic-policy-environment","text":"Go right? # Make the environment based on non-deterministic policy env = gym . make ( 'FrozenLake-v0' ) # Go right once (action = 2), we should go to the right but we did not! env . seed ( 8 ) env . reset () action = 2 ( observation , reward , done , prob ) = env . step ( action ) env . render () # Observation = 0: move to the right once from grid 0 to grid 1 # Prob = 1/3: non-deterministic policy, if we choose to go right, there's only a 1/3 probability we would go to the right and with this environment seed we did not print ( observation , reward , done , prob ) (Right) [S]FFF FHFH FFFH HFFG 0 0.0 False {'prob': 0.3333333333333333} Go right 10 times? # Try to go to the right 10 times, let's see how many times it goes to the right, by right we won't die because we would end up at the extreme right of grid 3 # See how it can go down/left/up/nothing instead of just right? # Intuitively when we are moving on a frozen lake, some times when we want to walk one direction we may end up in another direction as it's slippery # Setting seed here of the environment so you can reproduce my results, otherwise stochastic policy will yield different results for each run env . seed ( 8 ) env . reset () for i in range ( 10 ): action = 2 ( observation , reward , done , prob ) = env . step ( action ) env . render () (Right) [S]FFF FHFH FFFH HFFG (Right) S[F]FF FHFH FFFH HFFG (Right) SF[F]F FHFH FFFH HFFG (Right) SFFF FH[F]H FFFH HFFG (Right) SFFF FHF[H] FFFH HFFG (Right) SFFF FHF[4H] FFFH HFFG (Right) SFFF FHF[H] FFFH HFFG (Right) SFFF FHF[H] FFFH HFFG (Right) SFFF FHF[H] FFFH HFFG (Right) SFFF FHF[H] FFFH HFFG","title":"Non-deterministic Policy Environment"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#custom-frozen-lake-non-deterministic-policy-environment","text":"Because original code from OpenAI only allows us to run env.step(action) , this is challenging if we want to do some visualization of our state-value and action-value (q-value) functions for learning Hence, we'll be copying the whole code from OpenAI Frozen Lake implementation and adding just one line to make sure we can get P via self.P = P This code is not important, you can just copy it import sys from contextlib import closing import numpy as np from six import StringIO , b from gym import utils from gym.envs.toy_text import discrete LEFT = 0 DOWN = 1 RIGHT = 2 UP = 3 MAPS = { \"4x4\" : [ \"SFFF\" , \"FHFH\" , \"FFFH\" , \"HFFG\" ], \"8x8\" : [ \"SFFFFFFF\" , \"FFFFFFFF\" , \"FFFHFFFF\" , \"FFFFFHFF\" , \"FFFHFFFF\" , \"FHHFFFHF\" , \"FHFFHFHF\" , \"FFFHFFFG\" ], } # Generates a random valid map (one that has a path from start to goal) # @params size, size of each side of the grid # @prams p, probability that a tile is frozen def generate_random_map ( size = 8 , p = 0.8 ): valid = False #BFS to check that it's a valid path def is_valid ( arr , r = 0 , c = 0 ): if arr [ r ][ c ] == 'G' : return True tmp = arr [ r ][ c ] arr [ r ][ c ] = \"#\" if r + 1 < size and arr [ r + 1 ][ c ] not in '#H' : if is_valid ( arr , r + 1 , c ) == True : arr [ r ][ c ] = tmp return True if c + 1 < size and arr [ r ][ c + 1 ] not in '#H' : if is_valid ( arr , r , c + 1 ) == True : arr [ r ][ c ] = tmp return True if r - 1 >= 0 and arr [ r - 1 ][ c ] not in '#H' : if is_valid ( arr , r - 1 , c ) == True : arr [ r ][ c ] = tmp return True if c - 1 >= 0 and arr [ r ][ c - 1 ] not in '#H' : if is_valid ( arr , r , c - 1 ) == True : arr [ r ][ c ] = tmp return True arr [ r ][ c ] = tmp return False while not valid : p = min ( 1 , p ) res = np . random . choice ([ 'F' , 'H' ], ( size , size ), p = [ p , 1 - p ]) res [ 0 ][ 0 ] = 'S' res [ - 1 ][ - 1 ] = 'G' valid = is_valid ( res ) return [ \"\" . join ( x ) for x in res ] class FrozenLakeEnv ( discrete . DiscreteEnv ): \"\"\" Winter is here. You and your friends were tossing around a frisbee at the park when you made a wild throw that left the frisbee out in the middle of the lake. The water is mostly frozen, but there are a few holes where the ice has melted. If you step into one of those holes, you'll fall into the freezing water. At this time, there's an international frisbee shortage, so it's absolutely imperative that you navigate across the lake and retrieve the disc. However, the ice is slippery, so you won't always move in the direction you intend. The surface is described using a grid like the following SFFF FHFH FFFH HFFG S : starting point, safe F : frozen surface, safe H : hole, fall to your doom G : goal, where the frisbee is located The episode ends when you reach the goal or fall in a hole. You receive a reward of 1 if you reach the goal, and zero otherwise. \"\"\" metadata = { 'render.modes' : [ 'human' , 'ansi' ]} def __init__ ( self , desc = None , map_name = \"4x4\" , is_slippery = True ): if desc is None and map_name is None : desc = generate_random_map () elif desc is None : desc = MAPS [ map_name ] self . desc = desc = np . asarray ( desc , dtype = 'c' ) self . nrow , self . ncol = nrow , ncol = desc . shape self . reward_range = ( 0 , 1 ) nA = 4 nS = nrow * ncol isd = np . array ( desc == b 'S' ) . astype ( 'float64' ) . ravel () isd /= isd . sum () P = { s : { a : [] for a in range ( nA )} for s in range ( nS )} def to_s ( row , col ): return row * ncol + col def inc ( row , col , a ): if a == 0 : # left col = max ( col - 1 , 0 ) elif a == 1 : # down row = min ( row + 1 , nrow - 1 ) elif a == 2 : # right col = min ( col + 1 , ncol - 1 ) elif a == 3 : # up row = max ( row - 1 , 0 ) return ( row , col ) for row in range ( nrow ): for col in range ( ncol ): s = to_s ( row , col ) for a in range ( 4 ): li = P [ s ][ a ] letter = desc [ row , col ] if letter in b 'GH' : li . append (( 1.0 , s , 0 , True )) else : if is_slippery : for b in [( a - 1 ) % 4 , a , ( a + 1 ) % 4 ]: newrow , newcol = inc ( row , col , b ) newstate = to_s ( newrow , newcol ) newletter = desc [ newrow , newcol ] done = bytes ( newletter ) in b 'GH' rew = float ( newletter == b 'G' ) li . append (( 1.0 / 3.0 , newstate , rew , done )) else : newrow , newcol = inc ( row , col , a ) newstate = to_s ( newrow , newcol ) newletter = desc [ newrow , newcol ] done = bytes ( newletter ) in b 'GH' rew = float ( newletter == b 'G' ) li . append (( 1.0 , newstate , rew , done )) # New change because environment only allows step without # specific state for learning environment! self . P = P super ( FrozenLakeEnv , self ) . __init__ ( nS , nA , P , isd ) def render ( self , mode = 'human' ): outfile = StringIO () if mode == 'ansi' else sys . stdout row , col = self . s // self . ncol , self . s % self . ncol desc = self . desc . tolist () desc = [[ c . decode ( 'utf-8' ) for c in line ] for line in desc ] desc [ row ][ col ] = utils . colorize ( desc [ row ][ col ], \"red\" , highlight = True ) if self . lastaction is not None : outfile . write ( \" ( {} ) \\n \" . format ([ \"Left\" , \"Down\" , \"Right\" , \"Up\" ][ self . lastaction ])) else : outfile . write ( \" \\n \" ) outfile . write ( \" \\n \" . join ( '' . join ( line ) for line in desc ) + \" \\n \" ) if mode != 'human' : with closing ( outfile ): return outfile . getvalue ()","title":"Custom Frozen Lake Non-deterministic Policy Environment"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#policy-evaluation","text":"","title":"Policy Evaluation"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#transition-probability-function","text":"\\(\\mathcal{P}_{ss'}^a = \\mathcal{P}(s' \\vert s, a) = \\mathbb{P} [S_{t+1} = s' \\vert S_t = s, A_t = a]\\)","title":"Transition Probability Function"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#deterministic-environment","text":"There's no probability distribution, if you decide to go left you'll go left Hence in this example, given current_state = 8 and action = 0 which is left, we will end up with probability = 1 in new_state = 9 # Deterministic env = FrozenLakeEnv ( is_slippery = False ) current_state = 10 # State from S_n=16 State space action = 0 # Left action from A_n=4 Action space [( probability , new_state , reward , done )] = env . P [ current_state ][ action ] print ( 'Probability {} , New State {} ' . format ( probability , new_state )) Probability 1.0, New State 9","title":"Deterministic Environment"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#stochastic-environment","text":"Given \\(S_t = 10, A_t = 0\\) in a stochastic environment, the transition probability functions indicate you can end up in grid 6, 9, 14 each with \u2153 probability: \\(\\mathbb{P} [S_{t+1} = 6 \\vert S_t = 10, A_t = 0] = \\frac{1}{3}\\) \\(\\mathbb{P} [S_{t+1} = 9 \\vert S_t = 10, A_t = 0] = \\frac{1}{3}\\) \\(\\mathbb{P} [S_{t+1} = 14 \\vert S_t = 10, A_t = 0] = \\frac{1}{3}\\) # Stochastic env = FrozenLakeEnv ( is_slippery = True ) current_state = 10 # State from S_n=16 State space action = 0 # Left action from A_n=4 Action space env . P [ current_state ][ action ] [(0.3333333333333333, 6, 0.0, False), (0.3333333333333333, 9, 0.0, False), (0.3333333333333333, 14, 0.0, False)]","title":"Stochastic Environment"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#random-policy-function","text":"Random Policy function # Random policy generation def generate_random_policy ( S_n , A_n ): # return np.random.randint(A_n, size=(S_n, A_n)) return np . ones ([ S_n , A_n ]) / A_n # Given the total number of states S_n = 16 # For each state out of 16 states, we can take 4 actions # Since this is a stochastic environment, we'll initialize a policy to have equal probabilities 0.25 of doing each action each state policy = generate_random_policy ( S_n , A_n ) print ( policy . shape ) (16, 4) Policy plot import seaborn as sns import matplotlib.pyplot as plt % matplotlib inline plt . figure ( figsize = ( 5 , 16 )) sns . heatmap ( policy , cmap = \"YlGnBu\" , annot = True , cbar = False );","title":"Random Policy Function"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#policy-evaluation-function-comprising-state-value-function","text":"How: \\(\\mathcal{V}_{\\pi}(s) = \\sum_{a \\in \\mathcal{A}} \\pi(a | s) \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a \\big[\\mathcal{R}_s^a + \\gamma {V}_{\\pi}(s')\\big]\\) Simple code equation: Values of state given policy = sum ( action probability * transition probability * [reward + discount * value of new state] ) Aim: getting state-values import numpy as np def policy_evaluation ( env , policy , gamma = 1. , theta = 1e-8 ): r \"\"\"Policy evaluation function. Loop until state values stable, delta < theta. Returns V comprising values of states under given policy. Args: env (gym.env): OpenAI environment class instantiated and assigned to an object. policy (np.array): policy array to evaluate gamma (float): discount rate for rewards theta (float): tiny positive number, anything below it indicates value function convergence \"\"\" # 1. Create state-value array (16,) V = np . zeros ( S_n ) while True : delta = 0 # 2. Loop through states for s in range ( S_n ): Vs = 0 # 2.1 Loop through actions for the unique state # Given each state, we've 4 actions associated with different probabilities # 0.25 x 4 in this case, so we'll be looping 4 times (4 action probabilities) at each state for a , action_prob in enumerate ( policy [ s ]): # 2.1.1 Loop through to get transition probabilities, next state, rewards and whether the game ended for prob , next_state , reward , done in env . P [ s ][ a ]: # State-value function to get our values of states given policy Vs += action_prob * prob * ( reward + gamma * V [ next_state ]) # This simple equation allows us to stop this loop when we've converged # How do we know? The new value of the state is smaller than a tiny positive value we set # State value change is tiny compared to what we have so we just stop! delta = max ( delta , np . abs ( V [ s ] - Vs )) # 2.2 Update our state value for that state V [ s ] = Vs # 3. Stop policy evaluation if our state values changes are smaller than our tiny positive number if delta < theta : break return V # Generate random policy with equal probabilities of each action given any state rand_policy = generate_random_policy ( S_n , A_n ) # Evaluate the policy to get state values V = policy_evaluation ( env , rand_policy ) # Plot heatmap plt . figure ( figsize = ( 8 , 8 )) sns . heatmap ( V . reshape ( 4 , 4 ), cmap = \"YlGnBu\" , annot = True , cbar = False ); # This is our environment # Notice how the state values near the goal have higher values? # Those with \"H\" = hole, where you die if you step, have 0 values indicating those are bad areas to be in env . render () [S]FFF FHFH FFFH HFFG","title":"Policy Evaluation Function comprising State-value Function"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#policy-improvement","text":"","title":"Policy Improvement"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#action-value-q-value-function-from-state-value-function","text":"How: \\(\\mathcal{Q}_{\\pi}(s, a) = \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a \\big[ \\mathcal{R}_s^a + \\gamma \\mathcal{V}_{\\pi}(s') \\big]\\) Code equation Values of action = sum ( transition probability * [reward + discount * value of next state] ) Aim: getting q-values (action-values) def q_value ( env , V , s , gamma = 1 ): r \"\"\"Q-value (action-value) function from state-value function Returns Q values, values of actions. Args: env (gym.env): OpenAI environment class instantiated and assigned to an object. V (np.array): array of state-values obtained from policy evaluation function. s (integer): integer representing current state in the gridworld gamma (float): discount rate for rewards. \"\"\" # 1. Create q-value array for one state # We have 4 actions, so let's create an array with the size of 4 q = np . zeros ( A_n ) # 2. Loop through each action for a in range ( A_n ): # 2.1 For each action, we've our transition probabilities, next state, rewards and whether the game ended for prob , next_state , reward , done in env . P [ s ][ a ]: # 2.1.1 Get our action-values from state-values q [ a ] += prob * ( reward + gamma * V [ next_state ]) # Return action values return q # For every state, we've 4 actions, hence we've 16 x 4 q values Q = np . zeros ([ S_n , A_n ]) # Loop through each state out of 16 # For each state, we will get the 4 q-values associated with the 4 actions for s in range ( env . nS ): Q [ s ] = q_value ( env , V , s ) plt . figure ( figsize = ( 5 , 16 )) sns . heatmap ( Q , cmap = \"YlGnBu\" , annot = True , cbar = False ); # Notice how 13/14, those in the last row of the gridworld just before reaching the goal of finishing the game, their action values are large? env . render () [S]FFF FHFH FFFH HFFG","title":"Action-value (Q-value) function from State-value function"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#policy-improvement-function","text":"How: maximizing q-values per state by choosing actions with highest q-values Aim: get improved policy def policy_improvement ( env , V , gamma = 1. ): r \"\"\"Function to improve the policy by utilizing state values and action (q) values. Args: env (gym.env): OpenAI environment class instantiated and assigned to an objects V (np.array): array of state-values obtained from policy evaluation function gamma (float): discount of rewards \"\"\" # 1. Blank policy policy = np . zeros ([ env . nS , env . nA ]) / env . nA # 2. For each state in 16 states for s in range ( env . nS ): # 2.1 Get q values: q.shape returns (4,) q = q_value ( env , V , s , gamma ) # 2.2 Find best action based on max q-value # np.argwhere(q==np.max(q)) gives the position of largest q value # given array([0.00852356, 0.01163091, 0.0108613 , 0.01550788]), this would return array([[3]]) of shape (1, 1) # .flatten() reduces the shape to (1,) where we've array([3]) best_a = np . argwhere ( q == np . max ( q )) . flatten () # 2.3 One-hot encode best action and store into policy array's row for that state # In our case where the best action is array([3]), this would return # array([0., 0., 0., 1.]) where position 3 is the best action # Now we can store the best action into our policy policy [ s ] = np . sum ([ np . eye ( env . nA )[ i ] for i in best_a ], axis = 0 ) / len ( best_a ) return policy new_policy = policy_improvement ( env , V ) plt . figure ( figsize = ( 5 , 16 )) sns . heatmap ( new_policy , cmap = \"YlGnBu\" , annot = True , cbar = False ); # Compared to this equiprobable policy, the one above is making some improvements by maximizing q-values per state plt . figure ( figsize = ( 5 , 16 )) sns . heatmap ( rand_policy , cmap = \"YlGnBu\" , annot = True , cbar = False );","title":"Policy Improvement Function"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#policy-iteration-function","text":"How: loop through policy evaluation (get state-values) and policy improvement functions (use state-values to calculate q-values to improve policy) until optimal policy obtained Aim: improve policy until convergence Convergence: difference of state values between old and new policies is very small (less than theta, a very small positive number) import copy def policy_iteration ( env , gamma = 1 , theta = 1e-8 ): # 1. Create equiprobable policy where every state has 4 actions with equal probabilities as a starting policy policy = np . ones ([ env . nS , env . nA ]) / env . nA # 2. Loop through policy_evaluation and policy_improvement functions while True : # 2.1 Get state-values V = policy_evaluation ( env , policy , gamma , theta ) # 2.2 Get new policy by getting q-values and maximizing q-values per state to get best action per state new_policy = policy_improvement ( env , V ) # 2.3 Stop if the value function estimates for successive policies has converged if np . max ( abs ( policy_evaluation ( env , policy ) - policy_evaluation ( env , new_policy ))) < theta * 1e2 : break ; # 2.4 Replace policy with new policy policy = copy . copy ( new_policy ) return policy , V # obtain the optimal policy and optimal state-value function policy_pi , V_pi = policy_iteration ( env ) # Optimal policy (pi) # LEFT = 0, DOWN = 1, RIGHT = 2, UP = 3 plt . figure ( figsize = ( 5 , 16 )) sns . heatmap ( policy_pi , cmap = \"YlGnBu\" , annot = True , cbar = False , square = True ); # State values plt . figure ( figsize = ( 8 , 8 )) sns . heatmap ( V_pi . reshape ( 4 , 4 ), cmap = \"YlGnBu\" , annot = True , cbar = False , square = True ); # State values without policy improvement, just evaluation plt . figure ( figsize = ( 8 , 8 )) sns . heatmap ( V . reshape ( 4 , 4 ), cmap = \"YlGnBu\" , annot = True , cbar = False );","title":"Policy Iteration Function"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#value-iteration","text":"Alternative to policy iteration How: loop through to find optimal value function then get one-off policy Aim: improve value function until convergence Convergence: until difference in new and old state values are small (smaller than theta, small positive number) def value_iteration ( env , gamma = 1 , theta = 1e-8 ): # 1. Create state values of shape (16,) V = np . zeros ( env . nS ) # 2. Loop through q-value function until convergence while True : delta = 0 # 2.1 Loop through each state for s in range ( env . nS ): # 2.2 Archive old state value v = V [ s ] # 2.3 New state value = max of q-value V [ s ] = max ( q_value ( env , V , s , gamma )) delta = max ( delta , abs ( V [ s ] - v )) # 2.2 If state value changes small, converged if delta < theta : break # 3. Extract one-off policy with optimal state values policy = policy_improvement ( env , V , gamma ) return policy , V policy_vi , V_vi = value_iteration ( env ) # Optimal policy # LEFT = 0, DOWN = 1, RIGHT = 2, UP = 3 plt . figure ( figsize = ( 5 , 16 )) sns . heatmap ( policy_vi , cmap = \"YlGnBu\" , annot = True , cbar = False , square = True ); # State values plt . figure ( figsize = ( 8 , 8 )) sns . heatmap ( V_vi . reshape ( 4 , 4 ), cmap = \"YlGnBu\" , annot = True , cbar = False , square = True );","title":"Value iteration"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/rl_notes/","text":"","title":"Rl notes"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/supervised_to_rl/","text":"Supervised Learning to Reinforcement Learning \u00b6 Supervised Learning \u00b6 The tasks we've covered so far fall under the category of supervised learning Before, we have gone through 2 major tasks: classification and regression with labels Classification : we've a number of MNIST images, we take them as input and we use a neural network for a classification task where we use the ground truth (whether the digits are 0-9) to construct our cross entropy loss Classification loss function (cross entropy loss): \\(- \\sum^K_1 L_i log(S_i)\\) \\(K\\) : number of classes \\(L_i\\) : ground truth (label/target) of i-th class \\(S_i\\) : output of softmax for i-th class Regression : alternatively we go through a regression task of say, predicting a time-series, but we still have the ground truth we use to construct our loss function Regression loss function (mean squared error): \\(\\frac{1}{n} \\sum_{i=1}^n(\\hat y_i - y_i)^2\\) \\(\\hat{y}\\) : prediction \\(y\\) : ground truth (label/target) The key emphasis here is that we have mainly gone through supervised learning tasks that requires labels. Without them, we would not be able to properly construct our loss functions for us to do 2 critical steps (1) backpropagate to get our gradients and (2) gradient descent to update our weights with our gradients Loss functions We have covered 2 basic loss functions such as cross entropy loss (classification task) and mean squared error (regression task). However there are many more loss functions for both classification and regression task that will be covered in a separate section. For example, there are alternatives to mean squared error (MSE) like mean absolute error (MAE or L1 Loss), Smooth L1 Loss (less sensitive to outliers), quantile regression loss function (allowing confidence intervals) and many more. Reinforcement learning is not supervised learning \u00b6 One difference is that there is no ground truth (label/target) There is typically no label as to what is the definitively right prediction, we have to explore to find out what's \"right\" (essentially, the best possible prediction) Instead of minimizing a loss function comprising a target and a prediction (as with supervised learning), in reinforcement learning we are typically concerned with maximizing our reward function by trying different actions and exploring what those actions yield in an environment Let's use a simple game example of driving and not colliding with puppies crossing the road Agent Driver Environment 3 lane road, puppies crossing and the agent States Left, center or right lane Actions To move from one state to another Turn left, center or right Reward Feedback on whether action is good/bad, essentially the goal of the problem Colliding with the puppy: -10 points Too close to the puppy (scares the puppy): -2 points Safe distance from the puppy: 10 points Value function Defines what is good in the long-run as compared to rewards which is immediate after an action takes the agent to another state It's somewhat the discounted sum of the rewards the agent is expected to get Policy This defines how the agent acts in its states In this case, the agent might first collide with the puppy and learn it's bad (-10 points), then try not collide as the second action and still learn it's bad to be too close (-2 points) and finally as the third action learn to steer clear of puppies (+10 points) as it yields the largest reward Gradually it'll learn to drive at a safe distance from puppies to collect points (+10 points for safe distance) To do this, the agent needs to go try different actions and learn from its mistakes (trial and error), attempting to maximize its long-term 2 Distinguishing Properties of reinforcement learning \u00b6 Essentially, 2 distinguishing properties of reinforcement learning are: 1 (1) \"Trial-and-error search\" (2) \"Delayed reward\" In the next section, we'll be covering the terms we'll dive into these key terms through the lens of Markov Decision Processes (MDPs) and Bellman Equations. Richard S. Sutton and Andrew G. Barto. Reinforcement Learning: An Introduction; 2 nd Edition . 2017. \u21a9","title":"Supervised Learning to Reinforcement Learning (RL)"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/supervised_to_rl/#supervised-learning-to-reinforcement-learning","text":"","title":"Supervised Learning to Reinforcement Learning"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/supervised_to_rl/#supervised-learning","text":"The tasks we've covered so far fall under the category of supervised learning Before, we have gone through 2 major tasks: classification and regression with labels Classification : we've a number of MNIST images, we take them as input and we use a neural network for a classification task where we use the ground truth (whether the digits are 0-9) to construct our cross entropy loss Classification loss function (cross entropy loss): \\(- \\sum^K_1 L_i log(S_i)\\) \\(K\\) : number of classes \\(L_i\\) : ground truth (label/target) of i-th class \\(S_i\\) : output of softmax for i-th class Regression : alternatively we go through a regression task of say, predicting a time-series, but we still have the ground truth we use to construct our loss function Regression loss function (mean squared error): \\(\\frac{1}{n} \\sum_{i=1}^n(\\hat y_i - y_i)^2\\) \\(\\hat{y}\\) : prediction \\(y\\) : ground truth (label/target) The key emphasis here is that we have mainly gone through supervised learning tasks that requires labels. Without them, we would not be able to properly construct our loss functions for us to do 2 critical steps (1) backpropagate to get our gradients and (2) gradient descent to update our weights with our gradients Loss functions We have covered 2 basic loss functions such as cross entropy loss (classification task) and mean squared error (regression task). However there are many more loss functions for both classification and regression task that will be covered in a separate section. For example, there are alternatives to mean squared error (MSE) like mean absolute error (MAE or L1 Loss), Smooth L1 Loss (less sensitive to outliers), quantile regression loss function (allowing confidence intervals) and many more.","title":"Supervised Learning"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/supervised_to_rl/#reinforcement-learning-is-not-supervised-learning","text":"One difference is that there is no ground truth (label/target) There is typically no label as to what is the definitively right prediction, we have to explore to find out what's \"right\" (essentially, the best possible prediction) Instead of minimizing a loss function comprising a target and a prediction (as with supervised learning), in reinforcement learning we are typically concerned with maximizing our reward function by trying different actions and exploring what those actions yield in an environment Let's use a simple game example of driving and not colliding with puppies crossing the road Agent Driver Environment 3 lane road, puppies crossing and the agent States Left, center or right lane Actions To move from one state to another Turn left, center or right Reward Feedback on whether action is good/bad, essentially the goal of the problem Colliding with the puppy: -10 points Too close to the puppy (scares the puppy): -2 points Safe distance from the puppy: 10 points Value function Defines what is good in the long-run as compared to rewards which is immediate after an action takes the agent to another state It's somewhat the discounted sum of the rewards the agent is expected to get Policy This defines how the agent acts in its states In this case, the agent might first collide with the puppy and learn it's bad (-10 points), then try not collide as the second action and still learn it's bad to be too close (-2 points) and finally as the third action learn to steer clear of puppies (+10 points) as it yields the largest reward Gradually it'll learn to drive at a safe distance from puppies to collect points (+10 points for safe distance) To do this, the agent needs to go try different actions and learn from its mistakes (trial and error), attempting to maximize its long-term","title":"Reinforcement learning is not supervised learning"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/supervised_to_rl/#2-distinguishing-properties-of-reinforcement-learning","text":"Essentially, 2 distinguishing properties of reinforcement learning are: 1 (1) \"Trial-and-error search\" (2) \"Delayed reward\" In the next section, we'll be covering the terms we'll dive into these key terms through the lens of Markov Decision Processes (MDPs) and Bellman Equations. Richard S. Sutton and Andrew G. Barto. Reinforcement Learning: An Introduction; 2 nd Edition . 2017. \u21a9","title":"2 Distinguishing Properties of reinforcement learning"},{"location":"deep_learning/fromscratch/fromscratch_cnn/","text":"CNN from Scratch \u00b6 This is an implementation of a simple CNN (one convolutional function, one non-linear function, one max pooling function, one affine function and one softargmax function) for a 10-class MNIST classification task. Data Pull \u00b6 Imports \u00b6 from sklearn.datasets import fetch_openml from sklearn.utils import check_random_state from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler import matplotlib.pyplot as plt import pandas as pd import numpy as np import torch % matplotlib inline Load Data \u00b6 # Load data from https://www.openml.org/d/554 X , y = fetch_openml ( 'mnist_784' , version = 1 , return_X_y = True ) ### Plot Single Sample # Print dataset print ( f 'X shape: { X . shape } ' ) print ( f 'y shape: { y . shape } ' ) # Print size of resized single sample single_sample = X [ 0 , :] . reshape ( 28 , 28 ) print ( f 'Single sample shape: { single_sample . shape } ' ) # Plot single sample (M x N matrix) plt . gray () plt . matshow ( single_sample ) plt . show () X shape: (70000, 784) y shape: (70000,) Single sample shape: (28, 28) <Figure size 432x288 with 0 Axes> Train-Test Data Split \u00b6 # Train-test split train_samples = 60000 random_state = check_random_state ( 0 ) permutation = random_state . permutation ( X . shape [ 0 ]) X = X [ permutation ] y = y [ permutation ] X = X . reshape (( X . shape [ 0 ], - 1 )) X_train , X_test , y_train , y_test = train_test_split ( X , y , train_size = train_samples , test_size = 10000 ) scaler = StandardScaler () X_train = scaler . fit_transform ( X_train ) X_test = scaler . transform ( X_test ) # Print shapes print ( f 'Training shape { X_train . shape } ' ) print ( f 'Testing shape { X_test . shape } ' ) Training shape (60000, 784) Testing shape (10000, 784) Forwardpropagation \u00b6 Convolutional Layer \u00b6 class ConvolutionalLayer : def __init__ ( self , num_kernels , kernel_shape ): # Number of kernels: 1D self . num_kernels = num_kernels # Shape of kernels: 2D self . kernel_shape = kernel_shape self . k = self . kernel_shape [ 0 ] # Kernel weights: 3D self . kernels_theta = torch . randn ( self . num_kernels , self . kernel_shape [ 0 ], self . kernel_shape [ 1 ]) def slider ( self , inp ): ''' Sliding generator that yields square areas of shape (kernel_shape, kernel_shape) sliding across our input. This assumes valid padding (no padding) and step size 1. ''' h , w = inp . shape for h_idx in range ( h - ( self . k - 1 )): for w_idx in range ( w - ( self . k - 1 )): single_slide_area = inp [ h_idx :( h_idx + self . k ), w_idx :( w_idx + self . k )] yield single_slide_area , h_idx , w_idx def forward ( self , inp ): ''' Slides kernel across image doing an element-wise MM then summing. Results in forward pass of convolutional layer of shape (output shape, output shape, number of kernels). ''' # Input: 2D of (height, width) assert single_sample . dim () == 2 , f 'Input not 2D, given { single_sample . dim () } D' # Output via Valid Padding (No Padding): 3D of (height, width, number of kernels) _ , w = inp . shape # P = 0 p = 0 # O = ((W - K + 2P) / S) + 1 = (28 - 3 + 0) + 1 = 25 o = ( w - self . k ) + 1 # Print shapes print ( 'Padding shape: \\t ' , p ) print ( 'Output shape: \\t ' , o ) # Initialize blank tensor output = torch . zeros (( o , o , self . num_kernels )) # Iterate through region for single_slide_area , h_idx , w_idx in self . slider ( inp ): if h_idx == 0 and w_idx == 0 : print ( 'Region shape: \\t ' , list ( single_slide_area . shape )) print ( 'Kernel shape: \\t ' , list ( self . kernels_theta . shape )) print ( 'Single Slide: \\t ' , list ( output [ h_idx , w_idx ] . shape )) # Sum values with each element-wise matrix multiplication across each kernel # Instead of doing another loop of each kernel, you simply just do a element-wise MM # of the single slide area with all the kernels yield, then summing the patch output [ h_idx , w_idx ] = torch . sum ( single_slide_area * self . kernels_theta , axis = ( 1 , 2 )) # Pass through non-linearity (sigmoid): 1 / 1 + exp(-output) output = 1. / ( 1. + torch . exp ( - output )) return output # Convert numpy array to torch tensor single_sample = X [ 0 , :] . reshape ( 28 , 28 ) single_sample = torch . tensor ( single_sample ) print ( '=' * 50 ) print ( f 'Input shape: \\t { list ( single_sample . shape ) } ' ) print ( '=' * 50 ) # Forward: conv conv = ConvolutionalLayer ( num_kernels = 8 , kernel_shape = [ 5 , 5 ]) output = conv . forward ( single_sample ) print ( '=' * 50 ) print ( f 'Conv (f) shape: \\t { list ( output . shape ) } ' ) print ( '=' * 50 ) ================================================== Input shape: [28, 28] ================================================== Padding shape: 0 Output shape: 24 Region shape: [5, 5] Kernel shape: [8, 5, 5] Single Slide: [8] ================================================== Conv (f) shape: [24, 24, 8] ================================================== Max Pooling Layer \u00b6 class MaxPoolLayer : # O = ((W - K) / S) + 1 def __init__ ( self , pooling_kernel_shape ): # Assume simplicity of K = S then O = W / S self . k = pooling_kernel_shape def slider ( self , inp ): ''' Sliding generator that yields areas for max pooling. ''' h , w , _ = inp . shape output_size = int ( w / self . k ) # Assume S = K for h_idx in range ( output_size ): for w_idx in range ( output_size ): single_slide_area = inp [ h_idx * self . k : h_idx * self . k + self . k , w_idx * self . k : w_idx * self . k + self . k ] yield single_slide_area , h_idx , w_idx def forward ( self , inp ): ''' Performs a forward pass of the maxpool layer using the given input. Returns a 3d numpy array with dimensions (h / 2, w / 2, num_filters). - input is a 3d numpy array with dimensions (h, w, num_filters) ''' self . last_input = inp h , w , num_kernels = inp . shape output_size = int ( w / self . k ) # Assume S = K output = torch . zeros ( output_size , output_size , num_kernels ) for single_slide_area , h_idx , w_idx in self . slider ( inp ): single_slide_area = torch . flatten ( single_slide_area , start_dim = 0 , end_dim = 1 ) output [ h_idx , w_idx ] = torch . max ( single_slide_area , dim = 0 ) . values return output print ( '=' * 50 ) print ( f 'Input shape: \\t { list ( output . shape ) } ' ) print ( '=' * 50 ) # Forward: pool pool = MaxPoolLayer ( pooling_kernel_shape = 2 ) output = pool . forward ( output ) print ( '=' * 50 ) print ( f 'Pool (f) shape: \\t { list ( output . shape ) } ' ) print ( '=' * 50 ) ================================================== Input shape: [24, 24, 8] ================================================== ================================================== Pool (f) shape: [12, 12, 8] ================================================== Affine and Soft(arg)max Layer \u00b6 class AffineAndSoftmaxLayer : def __init__ ( self , affine_weight_shape ): self . affine_weight_shape = affine_weight_shape # Weight shape: flattened input x output shape self . w = torch . zeros ( self . affine_weight_shape [ 0 ] * self . affine_weight_shape [ 1 ] * self . affine_weight_shape [ 2 ], self . affine_weight_shape [ 3 ]) self . b = torch . zeros ( self . affine_weight_shape [ 3 ]) # Initialize weight/bias via Lecun initialization of 1 / N standard deviation # Refer to DLW guide on weight initialization mathematical derivation: # https://www.deeplearningwizard.com/deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/ print ( f 'Lecun initialization SD: { 1 / self . affine_weight_shape [ 3 ] } ' ) self . w = torch . nn . init . normal_ ( self . w , mean = 0 , std = 1 / self . affine_weight_shape [ 3 ]) self . b = torch . nn . init . normal_ ( self . b , mean = 0 , std = 1 / self . affine_weight_shape [ 3 ]) def forward ( self , inp ): ''' Performs Linear (Affine) Function & Soft(arg)max Function that returns our vector (1D) of probabilities. ''' inp = inp . reshape ( 1 , - 1 ) print ( f 'input shape: \\t { inp . shape } ' ) print ( f 'weight shape: \\t { self . w . shape } ' ) print ( f 'bias shape: \\t { self . b . shape } ' ) logits = torch . mm ( inp , self . w ) + self . b probas = torch . exp ( logits ) / torch . sum ( torch . exp ( logits )) return probas print ( '=' * 50 ) print ( f 'Input shape: \\t { list ( output . shape ) } ' ) print ( '=' * 50 ) # Forward: Affine and Softmax affinesoftmax = AffineAndSoftmaxLayer ( affine_weight_shape = ( output . shape [ 0 ], output . shape [ 1 ], output . shape [ 2 ], len ( np . unique ( y_train )))) output = affinesoftmax . forward ( output ) print ( '=' * 50 ) print ( f 'Affine & Soft(arg)max (f) shape: \\t { list ( output . shape ) } ' ) print ( '=' * 50 ) print ( f 'Probas: { pd . DataFrame ( output . numpy ()) . to_string ( index = False , header = False ) } ' ) ================================================== Input shape: [12, 12, 8] ================================================== Lecun initialization SD: 0.1 input shape: torch.Size([1, 1152]) weight shape: torch.Size([1152, 10]) bias shape: torch.Size([10]) ================================================== Affine & Soft(arg)max (f) shape: [1, 10] ================================================== Probas: 0.72574 0.001391 0.000083 0.202983 0.000503 0.037023 0.000428 0.000144 0.031293 0.000412 Dot Product, Matrix Multiplication, and Hadamard Product Hadamard product : element-wise multiplicaton of 2 matrices. Matrix Multiplication : take the first row of the first matrix and perform dot product with each of the N columns in the second matrix to form N columns in the first row of the new matrix. Repeat for remaining rows for the first matrix. # Backward & GD: linear + softmax # Backward & GD: pool # Backward & GD: conv Note on Contiguous vs Non-Contiguous \u00b6 Often in the space of passing tensors, and doing all our dot products, hadamard products, matrix multiplications, transpose, reshape operations and more, you would inevitably one day encounter into an error that says your tensor is not contiguous . This is a memory allocation problem. Certain tensor operations like transpose, view, expand, narrow etc do not change the original tensor, instead they modify the properties of the tensor. For example, transpose , demonstrated here, would change the shape (index), but both the old and modified property tensor share the same memory block with different indexes/addresses. This is why it's non-contiguous and we need to make it contiguous for some operations and typically for efficiency purpose. This is not a blanket statement, but you typically want your tensors to be contiguous as it prevents additional overhead incurred from translating addresses. Whenever there's a warning that prompts you the tensor is not contiguous, just call .contiguous() and you typically should be good to go. Contiguous 10 by 5 tensor \u00b6 contiguous_tensor = torch . arange ( 50 ) . view ( 10 , 5 ) print ( contiguous_tensor . shape ) # Pretty print quick hack via Pandas DataFrame print ( pd . DataFrame ( contiguous_tensor . numpy ()) . to_string ( index = False , header = False )) torch.Size([10, 5]) 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 Stride 5 by 1 \u00b6 Stride here shows how we need: 5 steps to move one row to the next & 1 step to move from one column to the next (contiguous) If this becomes anything other than 1 step, it becomes not contiguous print ( contiguous_tensor . stride ()) print ( contiguous_tensor . is_contiguous ()) (5, 1) True Non-contiguous Tensor via Transpose Operation \u00b6 In order to access the next \"column\" value of 5, we have to take 5 steps despite the transpose as if we would do in our original tensor Because the original tensor and this transposed tensor share the same memory block! non_contiguous_tensor = contiguous_tensor . t () print ( non_contiguous_tensor . shape ) print ( pd . DataFrame ( non_contiguous_tensor . numpy ()) . to_string ( index = False , header = False )) torch.Size([5, 10]) 0 5 10 15 20 25 30 35 40 45 1 6 11 16 21 26 31 36 41 46 2 7 12 17 22 27 32 37 42 47 3 8 13 18 23 28 33 38 43 48 4 9 14 19 24 29 34 39 44 49 Stride 1 by 5 \u00b6 print ( non_contiguous_tensor . stride ()) print ( non_contiguous_tensor . is_contiguous ()) (1, 5) False Convert to Contiguous \u00b6 # Convert to contiguous convert_contiguous = non_contiguous_tensor . contiguous () print ( convert_contiguous . is_contiguous ()) True","title":"CNN from Scratch"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#cnn-from-scratch","text":"This is an implementation of a simple CNN (one convolutional function, one non-linear function, one max pooling function, one affine function and one softargmax function) for a 10-class MNIST classification task.","title":"CNN from Scratch"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#data-pull","text":"","title":"Data Pull"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#imports","text":"from sklearn.datasets import fetch_openml from sklearn.utils import check_random_state from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler import matplotlib.pyplot as plt import pandas as pd import numpy as np import torch % matplotlib inline","title":"Imports"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#load-data","text":"# Load data from https://www.openml.org/d/554 X , y = fetch_openml ( 'mnist_784' , version = 1 , return_X_y = True ) ### Plot Single Sample # Print dataset print ( f 'X shape: { X . shape } ' ) print ( f 'y shape: { y . shape } ' ) # Print size of resized single sample single_sample = X [ 0 , :] . reshape ( 28 , 28 ) print ( f 'Single sample shape: { single_sample . shape } ' ) # Plot single sample (M x N matrix) plt . gray () plt . matshow ( single_sample ) plt . show () X shape: (70000, 784) y shape: (70000,) Single sample shape: (28, 28) <Figure size 432x288 with 0 Axes>","title":"Load Data"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#train-test-data-split","text":"# Train-test split train_samples = 60000 random_state = check_random_state ( 0 ) permutation = random_state . permutation ( X . shape [ 0 ]) X = X [ permutation ] y = y [ permutation ] X = X . reshape (( X . shape [ 0 ], - 1 )) X_train , X_test , y_train , y_test = train_test_split ( X , y , train_size = train_samples , test_size = 10000 ) scaler = StandardScaler () X_train = scaler . fit_transform ( X_train ) X_test = scaler . transform ( X_test ) # Print shapes print ( f 'Training shape { X_train . shape } ' ) print ( f 'Testing shape { X_test . shape } ' ) Training shape (60000, 784) Testing shape (10000, 784)","title":"Train-Test Data Split"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#forwardpropagation","text":"","title":"Forwardpropagation"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#convolutional-layer","text":"class ConvolutionalLayer : def __init__ ( self , num_kernels , kernel_shape ): # Number of kernels: 1D self . num_kernels = num_kernels # Shape of kernels: 2D self . kernel_shape = kernel_shape self . k = self . kernel_shape [ 0 ] # Kernel weights: 3D self . kernels_theta = torch . randn ( self . num_kernels , self . kernel_shape [ 0 ], self . kernel_shape [ 1 ]) def slider ( self , inp ): ''' Sliding generator that yields square areas of shape (kernel_shape, kernel_shape) sliding across our input. This assumes valid padding (no padding) and step size 1. ''' h , w = inp . shape for h_idx in range ( h - ( self . k - 1 )): for w_idx in range ( w - ( self . k - 1 )): single_slide_area = inp [ h_idx :( h_idx + self . k ), w_idx :( w_idx + self . k )] yield single_slide_area , h_idx , w_idx def forward ( self , inp ): ''' Slides kernel across image doing an element-wise MM then summing. Results in forward pass of convolutional layer of shape (output shape, output shape, number of kernels). ''' # Input: 2D of (height, width) assert single_sample . dim () == 2 , f 'Input not 2D, given { single_sample . dim () } D' # Output via Valid Padding (No Padding): 3D of (height, width, number of kernels) _ , w = inp . shape # P = 0 p = 0 # O = ((W - K + 2P) / S) + 1 = (28 - 3 + 0) + 1 = 25 o = ( w - self . k ) + 1 # Print shapes print ( 'Padding shape: \\t ' , p ) print ( 'Output shape: \\t ' , o ) # Initialize blank tensor output = torch . zeros (( o , o , self . num_kernels )) # Iterate through region for single_slide_area , h_idx , w_idx in self . slider ( inp ): if h_idx == 0 and w_idx == 0 : print ( 'Region shape: \\t ' , list ( single_slide_area . shape )) print ( 'Kernel shape: \\t ' , list ( self . kernels_theta . shape )) print ( 'Single Slide: \\t ' , list ( output [ h_idx , w_idx ] . shape )) # Sum values with each element-wise matrix multiplication across each kernel # Instead of doing another loop of each kernel, you simply just do a element-wise MM # of the single slide area with all the kernels yield, then summing the patch output [ h_idx , w_idx ] = torch . sum ( single_slide_area * self . kernels_theta , axis = ( 1 , 2 )) # Pass through non-linearity (sigmoid): 1 / 1 + exp(-output) output = 1. / ( 1. + torch . exp ( - output )) return output # Convert numpy array to torch tensor single_sample = X [ 0 , :] . reshape ( 28 , 28 ) single_sample = torch . tensor ( single_sample ) print ( '=' * 50 ) print ( f 'Input shape: \\t { list ( single_sample . shape ) } ' ) print ( '=' * 50 ) # Forward: conv conv = ConvolutionalLayer ( num_kernels = 8 , kernel_shape = [ 5 , 5 ]) output = conv . forward ( single_sample ) print ( '=' * 50 ) print ( f 'Conv (f) shape: \\t { list ( output . shape ) } ' ) print ( '=' * 50 ) ================================================== Input shape: [28, 28] ================================================== Padding shape: 0 Output shape: 24 Region shape: [5, 5] Kernel shape: [8, 5, 5] Single Slide: [8] ================================================== Conv (f) shape: [24, 24, 8] ==================================================","title":"Convolutional Layer"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#max-pooling-layer","text":"class MaxPoolLayer : # O = ((W - K) / S) + 1 def __init__ ( self , pooling_kernel_shape ): # Assume simplicity of K = S then O = W / S self . k = pooling_kernel_shape def slider ( self , inp ): ''' Sliding generator that yields areas for max pooling. ''' h , w , _ = inp . shape output_size = int ( w / self . k ) # Assume S = K for h_idx in range ( output_size ): for w_idx in range ( output_size ): single_slide_area = inp [ h_idx * self . k : h_idx * self . k + self . k , w_idx * self . k : w_idx * self . k + self . k ] yield single_slide_area , h_idx , w_idx def forward ( self , inp ): ''' Performs a forward pass of the maxpool layer using the given input. Returns a 3d numpy array with dimensions (h / 2, w / 2, num_filters). - input is a 3d numpy array with dimensions (h, w, num_filters) ''' self . last_input = inp h , w , num_kernels = inp . shape output_size = int ( w / self . k ) # Assume S = K output = torch . zeros ( output_size , output_size , num_kernels ) for single_slide_area , h_idx , w_idx in self . slider ( inp ): single_slide_area = torch . flatten ( single_slide_area , start_dim = 0 , end_dim = 1 ) output [ h_idx , w_idx ] = torch . max ( single_slide_area , dim = 0 ) . values return output print ( '=' * 50 ) print ( f 'Input shape: \\t { list ( output . shape ) } ' ) print ( '=' * 50 ) # Forward: pool pool = MaxPoolLayer ( pooling_kernel_shape = 2 ) output = pool . forward ( output ) print ( '=' * 50 ) print ( f 'Pool (f) shape: \\t { list ( output . shape ) } ' ) print ( '=' * 50 ) ================================================== Input shape: [24, 24, 8] ================================================== ================================================== Pool (f) shape: [12, 12, 8] ==================================================","title":"Max Pooling Layer"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#affine-and-softargmax-layer","text":"class AffineAndSoftmaxLayer : def __init__ ( self , affine_weight_shape ): self . affine_weight_shape = affine_weight_shape # Weight shape: flattened input x output shape self . w = torch . zeros ( self . affine_weight_shape [ 0 ] * self . affine_weight_shape [ 1 ] * self . affine_weight_shape [ 2 ], self . affine_weight_shape [ 3 ]) self . b = torch . zeros ( self . affine_weight_shape [ 3 ]) # Initialize weight/bias via Lecun initialization of 1 / N standard deviation # Refer to DLW guide on weight initialization mathematical derivation: # https://www.deeplearningwizard.com/deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/ print ( f 'Lecun initialization SD: { 1 / self . affine_weight_shape [ 3 ] } ' ) self . w = torch . nn . init . normal_ ( self . w , mean = 0 , std = 1 / self . affine_weight_shape [ 3 ]) self . b = torch . nn . init . normal_ ( self . b , mean = 0 , std = 1 / self . affine_weight_shape [ 3 ]) def forward ( self , inp ): ''' Performs Linear (Affine) Function & Soft(arg)max Function that returns our vector (1D) of probabilities. ''' inp = inp . reshape ( 1 , - 1 ) print ( f 'input shape: \\t { inp . shape } ' ) print ( f 'weight shape: \\t { self . w . shape } ' ) print ( f 'bias shape: \\t { self . b . shape } ' ) logits = torch . mm ( inp , self . w ) + self . b probas = torch . exp ( logits ) / torch . sum ( torch . exp ( logits )) return probas print ( '=' * 50 ) print ( f 'Input shape: \\t { list ( output . shape ) } ' ) print ( '=' * 50 ) # Forward: Affine and Softmax affinesoftmax = AffineAndSoftmaxLayer ( affine_weight_shape = ( output . shape [ 0 ], output . shape [ 1 ], output . shape [ 2 ], len ( np . unique ( y_train )))) output = affinesoftmax . forward ( output ) print ( '=' * 50 ) print ( f 'Affine & Soft(arg)max (f) shape: \\t { list ( output . shape ) } ' ) print ( '=' * 50 ) print ( f 'Probas: { pd . DataFrame ( output . numpy ()) . to_string ( index = False , header = False ) } ' ) ================================================== Input shape: [12, 12, 8] ================================================== Lecun initialization SD: 0.1 input shape: torch.Size([1, 1152]) weight shape: torch.Size([1152, 10]) bias shape: torch.Size([10]) ================================================== Affine & Soft(arg)max (f) shape: [1, 10] ================================================== Probas: 0.72574 0.001391 0.000083 0.202983 0.000503 0.037023 0.000428 0.000144 0.031293 0.000412 Dot Product, Matrix Multiplication, and Hadamard Product Hadamard product : element-wise multiplicaton of 2 matrices. Matrix Multiplication : take the first row of the first matrix and perform dot product with each of the N columns in the second matrix to form N columns in the first row of the new matrix. Repeat for remaining rows for the first matrix. # Backward & GD: linear + softmax # Backward & GD: pool # Backward & GD: conv","title":"Affine and Soft(arg)max Layer"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#note-on-contiguous-vs-non-contiguous","text":"Often in the space of passing tensors, and doing all our dot products, hadamard products, matrix multiplications, transpose, reshape operations and more, you would inevitably one day encounter into an error that says your tensor is not contiguous . This is a memory allocation problem. Certain tensor operations like transpose, view, expand, narrow etc do not change the original tensor, instead they modify the properties of the tensor. For example, transpose , demonstrated here, would change the shape (index), but both the old and modified property tensor share the same memory block with different indexes/addresses. This is why it's non-contiguous and we need to make it contiguous for some operations and typically for efficiency purpose. This is not a blanket statement, but you typically want your tensors to be contiguous as it prevents additional overhead incurred from translating addresses. Whenever there's a warning that prompts you the tensor is not contiguous, just call .contiguous() and you typically should be good to go.","title":"Note on Contiguous vs Non-Contiguous"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#contiguous-10-by-5-tensor","text":"contiguous_tensor = torch . arange ( 50 ) . view ( 10 , 5 ) print ( contiguous_tensor . shape ) # Pretty print quick hack via Pandas DataFrame print ( pd . DataFrame ( contiguous_tensor . numpy ()) . to_string ( index = False , header = False )) torch.Size([10, 5]) 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49","title":"Contiguous 10 by 5 tensor"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#stride-5-by-1","text":"Stride here shows how we need: 5 steps to move one row to the next & 1 step to move from one column to the next (contiguous) If this becomes anything other than 1 step, it becomes not contiguous print ( contiguous_tensor . stride ()) print ( contiguous_tensor . is_contiguous ()) (5, 1) True","title":"Stride 5 by 1"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#non-contiguous-tensor-via-transpose-operation","text":"In order to access the next \"column\" value of 5, we have to take 5 steps despite the transpose as if we would do in our original tensor Because the original tensor and this transposed tensor share the same memory block! non_contiguous_tensor = contiguous_tensor . t () print ( non_contiguous_tensor . shape ) print ( pd . DataFrame ( non_contiguous_tensor . numpy ()) . to_string ( index = False , header = False )) torch.Size([5, 10]) 0 5 10 15 20 25 30 35 40 45 1 6 11 16 21 26 31 36 41 46 2 7 12 17 22 27 32 37 42 47 3 8 13 18 23 28 33 38 43 48 4 9 14 19 24 29 34 39 44 49","title":"Non-contiguous Tensor via Transpose Operation"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#stride-1-by-5","text":"print ( non_contiguous_tensor . stride ()) print ( non_contiguous_tensor . is_contiguous ()) (1, 5) False","title":"Stride 1 by 5"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#convert-to-contiguous","text":"# Convert to contiguous convert_contiguous = non_contiguous_tensor . contiguous () print ( convert_contiguous . is_contiguous ()) True","title":"Convert to Contiguous"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/","text":"Logistic Regression from Scratch \u00b6 This is an implementation of a simple logistic regression for binary class labels. We will be attempting to classify 2 flowers based on their petal width and height: setosa and versicolor. Imports \u00b6 % matplotlib inline import matplotlib.pyplot as plt import numpy as np import torch import torch.nn.functional as F from sklearn import datasets from sklearn import preprocessing from sklearn.model_selection import train_test_split from collections import Counter Preparing a custom 2-class IRIS dataset \u00b6 Load Data \u00b6 # Instantiate dataset class and assign to object iris = datasets . load_iris () # Load features and target # Take only 2 classes, and 2 features (sepal length/width) X = iris . data [: - 50 , : 2 ] # For teaching the math rather than preprocessing techniques, # we'll be using this simple scaling method. However, you must # be cautious to scale your training/testing sets subsequently. X = preprocessing . scale ( X ) y = iris . target [: - 50 ] Print Data Details \u00b6 # 50 of each iris flower print ( Counter ( y )) # Type of flower print ( list ( iris . target_names [: - 1 ])) # Shape of features print ( X . shape ) Counter({0: 50, 1: 50}) ['setosa', 'versicolor'] (100, 2) Scatterplot 2 Classes \u00b6 plt . scatter ( X [:, 0 ], X [:, 1 ], c = y ); Train/Test Split \u00b6 X_train , X_test , y_train , y_test = train_test_split ( X , y , test_size = 0.2 ) print ( f 'X train size: { X_train . shape } ' ) print ( f 'X test size: { X_test . shape } ' ) print ( f 'y train size: { y_train . shape } ' ) print ( f 'y test size: { y_test . shape } ' ) # Distribution of both classes are roughly equal using train_test_split function print ( Counter ( y_train )) X train size: (80, 2) X test size: (20, 2) y train size: (80,) y test size: (20,) Counter({0: 41, 1: 39}) Math \u00b6 1. Forwardpropagation \u00b6 Get our logits and probabilities Affine function/transformation: \\(z = \\theta x + b\\) Sigmoid/logistic function: \\(\\hat y = \\frac{1}{1 + e^{-z}}\\) 2. Backwardpropagation \u00b6 Calculate gradients / partial derivatives w.r.t. weights and bias Loss: \\(L = ylog(\\hat y) + (1-y) log (1 - \\hat y)\\) Partial derivative of loss w.r.t weights: \\(\\frac{\\delta L}{\\delta w} =\\frac{\\delta L}{\\delta z} \\frac{\\delta z}{\\delta w} = (\\hat y - y)(x^T)\\) Partial derivative of loss w.r.t. bias: \\(\\frac{\\delta L}{\\delta b} = \\frac{\\delta L}{\\delta z} \\frac{\\delta z}{\\delta b} = (\\hat y - y)(1)\\) \\(\\frac{\\delta L}{\\delta z} = \\hat y - y\\) \\(\\frac{\\delta z}{\\delta w} = x\\) \\(\\frac{\\delta z}{\\delta b} = 1\\) 2a. Loss function clarification \u00b6 Actually, why is our loss equation \\(L = ylog(\\hat y) + (1-y) log (1 - \\hat y)\\) ? We have given the intuition in the Logistic Regression tutorial on why it works. Here we will cover the derivation which essentially is merely maximizing the log likelihood of the parameters (maximizing the probability of our predicted output given our input and parameters Given: \\(\\hat y = \\frac{1}{1 + e^{-z}}\\) . Then: \\(P(y=1 \\mid x;\\theta) = \\hat y\\) \\(P(y=0 \\mid x;\\theta) = 1 - \\hat y\\) Simplified further: \\(p(y \\mid x; \\theta) = (\\hat y)^y(1 - \\hat y)^{1-y}\\) Given m training samples, the likelihood of the parameters is simply the product of probabilities: \\(L(\\theta) = \\displaystyle \\prod_{i=1}^{m} p(y^i \\mid x^i; \\theta)\\) \\(L(\\theta) = \\displaystyle \\prod_{i=1}^{m} (\\hat y^{i})^{y^i}(1 - \\hat y^{i})^{1-y^{i}}\\) Essentially, we want to maximize the probability of our ouput given our input and parameters But it's easier to maximize the log likelihood, so we take the natural logarithm. \\(L(\\theta) = \\displaystyle \\sum_{i=1}^{m} y^{i}log (\\hat y^{i}) + (1 - y^{i})log(1 - \\hat y^{i})\\) Why is is easier to maximize the log likelihood? The natural logarithm is a function that monotonically increases. This allows us to find the \"max\" of the log likelihood easier compared to a non-monotonically increasing function (like a wave up and down). 3. Gradient descent: updating weights \u00b6 \\(w = w - \\alpha (\\hat y - y)(x^T)\\) \\(b = b - \\alpha (\\hat y - y).1\\) Training from Scratch \u00b6 learning_rate = 0.1 num_features = X . shape [ 1 ] weights = torch . zeros ( num_features , 1 , dtype = torch . float32 ) bias = torch . zeros ( 1 , dtype = torch . float32 ) X_train = torch . from_numpy ( X_train ) . type ( torch . float32 ) y_train = torch . from_numpy ( y_train ) . type ( torch . float32 ) for epoch in range ( num_epochs ): # 1. Forwardpropagation: # 1a. Affine Transformation: z = \\theta x + b z = torch . add ( torch . mm ( X_train , weights ), bias ) # 2a. Sigmoid/Logistic Function: y_hat = 1 / (1 + e^{-z}) y_hat = 1. / ( 1. + torch . exp ( - z )) # Backpropagation: # 1. Calculate binary cross entropy l = torch . mm ( - y_train . view ( 1 , - 1 ), torch . log ( y_hat )) - torch . mm (( 1 - y_train ) . view ( 1 , - 1 ), torch . log ( 1 - y_hat )) # 2. Calculate dl/dz dl_dz = y_train - y_hat . view ( - 1 ) # 2. Calculate partial derivative of cost w.r.t weights (gradients) # dl_dw = dl_dz dz_dw = (y_hat - y)(x^T) grad = torch . mm ( X_train . transpose ( 0 , 1 ), dl_dz . view ( - 1 , 1 )) # Gradient descent: # update our weights and bias with our gradients weights += learning_rate * grad bias += learning_rate * torch . sum ( dl_dz ) # Accuracy total = y_hat . shape [ 0 ] predicted = ( y_hat > 0.5 ) . float () . squeeze () correct = ( predicted == y_train ) . sum () acc = 100 * correct / total # Print accuracy and cost print ( f 'Epoch: { epoch } | Accuracy: { acc . item () : .4f } | Cost: { l . item () : .4f } ' ) print ( f 'Weights \\n { weights . data } ' ) print ( f 'Bias \\n { bias . data } ' ) X train size: (80, 2) X test size: (20, 2) y train size: (80,) y test size: (20,) Counter({1: 41, 0: 39}) Epoch: 0 | Accuracy: 48.0000 | Cost: 55.4518 Epoch: 1 | Accuracy: 100.0000 | Cost: 5.6060 Epoch: 2 | Accuracy: 100.0000 | Cost: 5.0319 Epoch: 3 | Accuracy: 100.0000 | Cost: 4.6001 Epoch: 4 | Accuracy: 100.0000 | Cost: 4.2595 Epoch: 5 | Accuracy: 100.0000 | Cost: 3.9819 Epoch: 6 | Accuracy: 100.0000 | Cost: 3.7498 Epoch: 7 | Accuracy: 100.0000 | Cost: 3.5521 Epoch: 8 | Accuracy: 100.0000 | Cost: 3.3810 Epoch: 9 | Accuracy: 100.0000 | Cost: 3.2310 Epoch: 10 | Accuracy: 100.0000 | Cost: 3.0981 Epoch: 11 | Accuracy: 100.0000 | Cost: 2.9794 Epoch: 12 | Accuracy: 100.0000 | Cost: 2.8724 Epoch: 13 | Accuracy: 100.0000 | Cost: 2.7754 Epoch: 14 | Accuracy: 100.0000 | Cost: 2.6869 Epoch: 15 | Accuracy: 100.0000 | Cost: 2.6057 Epoch: 16 | Accuracy: 100.0000 | Cost: 2.5308 Epoch: 17 | Accuracy: 100.0000 | Cost: 2.4616 Epoch: 18 | Accuracy: 100.0000 | Cost: 2.3973 Epoch: 19 | Accuracy: 100.0000 | Cost: 2.3374 Weights tensor([[ 4.9453], [-3.6849]]) Bias tensor([0.5570]) Inference \u00b6 # Port to tensors X_test = torch . from_numpy ( X_test ) . type ( torch . float32 ) y_test = torch . from_numpy ( y_test ) . type ( torch . float32 ) # 1. Forwardpropagation: # 1a. Affine Transformation: z = ax + b z = torch . add ( torch . mm ( X_test , weights ), bias ) # 2a. Sigmoid/Logistic Function: y_hat = 1 / (1 + e^{-z}) y_hat = 1. / ( 1. + torch . exp ( - z )) total = y_test . shape [ 0 ] predicted = ( y_hat > 0.5 ) . float () . squeeze () correct = ( predicted == y_test ) . sum () acc = 100 * correct / total # Print accuracy print ( f 'Validation Accuracy: { acc . item () : .4f } ' ) Validation Accuracy: 100.0000","title":"From Scratch Logistic Regression Classification"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#logistic-regression-from-scratch","text":"This is an implementation of a simple logistic regression for binary class labels. We will be attempting to classify 2 flowers based on their petal width and height: setosa and versicolor.","title":"Logistic Regression from Scratch"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#imports","text":"% matplotlib inline import matplotlib.pyplot as plt import numpy as np import torch import torch.nn.functional as F from sklearn import datasets from sklearn import preprocessing from sklearn.model_selection import train_test_split from collections import Counter","title":"Imports"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#preparing-a-custom-2-class-iris-dataset","text":"","title":"Preparing a custom 2-class IRIS dataset"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#load-data","text":"# Instantiate dataset class and assign to object iris = datasets . load_iris () # Load features and target # Take only 2 classes, and 2 features (sepal length/width) X = iris . data [: - 50 , : 2 ] # For teaching the math rather than preprocessing techniques, # we'll be using this simple scaling method. However, you must # be cautious to scale your training/testing sets subsequently. X = preprocessing . scale ( X ) y = iris . target [: - 50 ]","title":"Load Data"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#print-data-details","text":"# 50 of each iris flower print ( Counter ( y )) # Type of flower print ( list ( iris . target_names [: - 1 ])) # Shape of features print ( X . shape ) Counter({0: 50, 1: 50}) ['setosa', 'versicolor'] (100, 2)","title":"Print Data Details"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#scatterplot-2-classes","text":"plt . scatter ( X [:, 0 ], X [:, 1 ], c = y );","title":"Scatterplot 2 Classes"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#traintest-split","text":"X_train , X_test , y_train , y_test = train_test_split ( X , y , test_size = 0.2 ) print ( f 'X train size: { X_train . shape } ' ) print ( f 'X test size: { X_test . shape } ' ) print ( f 'y train size: { y_train . shape } ' ) print ( f 'y test size: { y_test . shape } ' ) # Distribution of both classes are roughly equal using train_test_split function print ( Counter ( y_train )) X train size: (80, 2) X test size: (20, 2) y train size: (80,) y test size: (20,) Counter({0: 41, 1: 39})","title":"Train/Test Split"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#math","text":"","title":"Math"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#1-forwardpropagation","text":"Get our logits and probabilities Affine function/transformation: \\(z = \\theta x + b\\) Sigmoid/logistic function: \\(\\hat y = \\frac{1}{1 + e^{-z}}\\)","title":"1. Forwardpropagation"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#2-backwardpropagation","text":"Calculate gradients / partial derivatives w.r.t. weights and bias Loss: \\(L = ylog(\\hat y) + (1-y) log (1 - \\hat y)\\) Partial derivative of loss w.r.t weights: \\(\\frac{\\delta L}{\\delta w} =\\frac{\\delta L}{\\delta z} \\frac{\\delta z}{\\delta w} = (\\hat y - y)(x^T)\\) Partial derivative of loss w.r.t. bias: \\(\\frac{\\delta L}{\\delta b} = \\frac{\\delta L}{\\delta z} \\frac{\\delta z}{\\delta b} = (\\hat y - y)(1)\\) \\(\\frac{\\delta L}{\\delta z} = \\hat y - y\\) \\(\\frac{\\delta z}{\\delta w} = x\\) \\(\\frac{\\delta z}{\\delta b} = 1\\)","title":"2. Backwardpropagation"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#2a-loss-function-clarification","text":"Actually, why is our loss equation \\(L = ylog(\\hat y) + (1-y) log (1 - \\hat y)\\) ? We have given the intuition in the Logistic Regression tutorial on why it works. Here we will cover the derivation which essentially is merely maximizing the log likelihood of the parameters (maximizing the probability of our predicted output given our input and parameters Given: \\(\\hat y = \\frac{1}{1 + e^{-z}}\\) . Then: \\(P(y=1 \\mid x;\\theta) = \\hat y\\) \\(P(y=0 \\mid x;\\theta) = 1 - \\hat y\\) Simplified further: \\(p(y \\mid x; \\theta) = (\\hat y)^y(1 - \\hat y)^{1-y}\\) Given m training samples, the likelihood of the parameters is simply the product of probabilities: \\(L(\\theta) = \\displaystyle \\prod_{i=1}^{m} p(y^i \\mid x^i; \\theta)\\) \\(L(\\theta) = \\displaystyle \\prod_{i=1}^{m} (\\hat y^{i})^{y^i}(1 - \\hat y^{i})^{1-y^{i}}\\) Essentially, we want to maximize the probability of our ouput given our input and parameters But it's easier to maximize the log likelihood, so we take the natural logarithm. \\(L(\\theta) = \\displaystyle \\sum_{i=1}^{m} y^{i}log (\\hat y^{i}) + (1 - y^{i})log(1 - \\hat y^{i})\\) Why is is easier to maximize the log likelihood? The natural logarithm is a function that monotonically increases. This allows us to find the \"max\" of the log likelihood easier compared to a non-monotonically increasing function (like a wave up and down).","title":"2a. Loss function clarification"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#3-gradient-descent-updating-weights","text":"\\(w = w - \\alpha (\\hat y - y)(x^T)\\) \\(b = b - \\alpha (\\hat y - y).1\\)","title":"3. Gradient descent: updating weights"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#training-from-scratch","text":"learning_rate = 0.1 num_features = X . shape [ 1 ] weights = torch . zeros ( num_features , 1 , dtype = torch . float32 ) bias = torch . zeros ( 1 , dtype = torch . float32 ) X_train = torch . from_numpy ( X_train ) . type ( torch . float32 ) y_train = torch . from_numpy ( y_train ) . type ( torch . float32 ) for epoch in range ( num_epochs ): # 1. Forwardpropagation: # 1a. Affine Transformation: z = \\theta x + b z = torch . add ( torch . mm ( X_train , weights ), bias ) # 2a. Sigmoid/Logistic Function: y_hat = 1 / (1 + e^{-z}) y_hat = 1. / ( 1. + torch . exp ( - z )) # Backpropagation: # 1. Calculate binary cross entropy l = torch . mm ( - y_train . view ( 1 , - 1 ), torch . log ( y_hat )) - torch . mm (( 1 - y_train ) . view ( 1 , - 1 ), torch . log ( 1 - y_hat )) # 2. Calculate dl/dz dl_dz = y_train - y_hat . view ( - 1 ) # 2. Calculate partial derivative of cost w.r.t weights (gradients) # dl_dw = dl_dz dz_dw = (y_hat - y)(x^T) grad = torch . mm ( X_train . transpose ( 0 , 1 ), dl_dz . view ( - 1 , 1 )) # Gradient descent: # update our weights and bias with our gradients weights += learning_rate * grad bias += learning_rate * torch . sum ( dl_dz ) # Accuracy total = y_hat . shape [ 0 ] predicted = ( y_hat > 0.5 ) . float () . squeeze () correct = ( predicted == y_train ) . sum () acc = 100 * correct / total # Print accuracy and cost print ( f 'Epoch: { epoch } | Accuracy: { acc . item () : .4f } | Cost: { l . item () : .4f } ' ) print ( f 'Weights \\n { weights . data } ' ) print ( f 'Bias \\n { bias . data } ' ) X train size: (80, 2) X test size: (20, 2) y train size: (80,) y test size: (20,) Counter({1: 41, 0: 39}) Epoch: 0 | Accuracy: 48.0000 | Cost: 55.4518 Epoch: 1 | Accuracy: 100.0000 | Cost: 5.6060 Epoch: 2 | Accuracy: 100.0000 | Cost: 5.0319 Epoch: 3 | Accuracy: 100.0000 | Cost: 4.6001 Epoch: 4 | Accuracy: 100.0000 | Cost: 4.2595 Epoch: 5 | Accuracy: 100.0000 | Cost: 3.9819 Epoch: 6 | Accuracy: 100.0000 | Cost: 3.7498 Epoch: 7 | Accuracy: 100.0000 | Cost: 3.5521 Epoch: 8 | Accuracy: 100.0000 | Cost: 3.3810 Epoch: 9 | Accuracy: 100.0000 | Cost: 3.2310 Epoch: 10 | Accuracy: 100.0000 | Cost: 3.0981 Epoch: 11 | Accuracy: 100.0000 | Cost: 2.9794 Epoch: 12 | Accuracy: 100.0000 | Cost: 2.8724 Epoch: 13 | Accuracy: 100.0000 | Cost: 2.7754 Epoch: 14 | Accuracy: 100.0000 | Cost: 2.6869 Epoch: 15 | Accuracy: 100.0000 | Cost: 2.6057 Epoch: 16 | Accuracy: 100.0000 | Cost: 2.5308 Epoch: 17 | Accuracy: 100.0000 | Cost: 2.4616 Epoch: 18 | Accuracy: 100.0000 | Cost: 2.3973 Epoch: 19 | Accuracy: 100.0000 | Cost: 2.3374 Weights tensor([[ 4.9453], [-3.6849]]) Bias tensor([0.5570])","title":"Training from Scratch"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#inference","text":"# Port to tensors X_test = torch . from_numpy ( X_test ) . type ( torch . float32 ) y_test = torch . from_numpy ( y_test ) . type ( torch . float32 ) # 1. Forwardpropagation: # 1a. Affine Transformation: z = ax + b z = torch . add ( torch . mm ( X_test , weights ), bias ) # 2a. Sigmoid/Logistic Function: y_hat = 1 / (1 + e^{-z}) y_hat = 1. / ( 1. + torch . exp ( - z )) total = y_test . shape [ 0 ] predicted = ( y_hat > 0.5 ) . float () . squeeze () correct = ( predicted == y_test ) . sum () acc = 100 * correct / total # Print accuracy print ( f 'Validation Accuracy: { acc . item () : .4f } ' ) Validation Accuracy: 100.0000","title":"Inference"},{"location":"deep_learning/practical_pytorch/pytorch_autoencoder/","text":"Autoencoders with PyTorch \u00b6 Run Jupyter Notebook You can run the code for this section in this jupyter notebook link . About Autoencoders \u00b6 Feedforward Neural Network (FNN) to Autoencoders (AEs) \u00b6 Autoencoder is a form of unsupervised learning . This is a big deviation from what we have been doing: classification and regression which are under supervised learning. There are no labels required, inputs are used as labels. This is a bit mind-boggling for some, but there're many conrete use cases as you'll soon realize. Just a quick preview of use cases we will be covering: Denoising overcomplete AEs: recreate images without the random noises originally injected. Undercomplete AEs for anomaly detection: use AEs for credit card fraud detection via anomaly detection. Variational AEs for creating synthetic faces: with a convolutional VAEs, we can make fake faces. An autoencoder's purpose is to learn an approximation of the identity function (mapping \\(x\\) to \\(\\hat x\\) ). Essentially we are trying to learn a function that can take our input \\(x\\) and recreate it \\(\\hat x\\) . Technically we can do an exact recreation of our in-sample input if we use a very wide and deep neural network. Undercomplete and Overcomplete Autoencoders \u00b6 When we highlighted some use cases, did you notice how we mentioned undercomplete and autocomplete AEs? The only difference between the two is in the encoding output's size. In the diagram above, this refers to the encoding output's size after our first affine function (yellow box) and non-linear function (pink box). Undercomplete AEs: smaller This is when our encoding output's dimension is smaller than our input's dimension. Essentially we reduced the dimension of our data (dimensionality reduction) with an undercomplete AE Overcomplete AEs: larger This is when our encoding output's dimension is larger than our input's dimension Essentially we increased the dimension of our data with an overcomplete AE Fully-connected and Convolutional Autoencoders \u00b6 Another important point is that, in our diagram we've used the example of our Feedforward Neural Networks (FNN) where we use fully-connected layers. This is called Fully-connected AE. However, we can easily swap those fully-connected layers with convolutional layers. This is called Convolutional AE. Autoencoders Series \u00b6 We'll be covering a series of autoencoders in this order Fully-connected Overcomplete Autoencoder (AEs): Denoising Images Fully-connected Undercomplete Autoencoder (AEs): Credit Card Fraud Detection Convolutional Overcomplete Variational Autoencoder (VAEs): Generate Fake Human Faces Convolutional Overcomplete Adversarial Autoencoder (AAEs): Generate Fake Human Faces Generative Adversarial Networks (GANs): Generate Better Fake Human Faces","title":"Autoencoders (AE)"},{"location":"deep_learning/practical_pytorch/pytorch_autoencoder/#autoencoders-with-pytorch","text":"Run Jupyter Notebook You can run the code for this section in this jupyter notebook link .","title":"Autoencoders with PyTorch"},{"location":"deep_learning/practical_pytorch/pytorch_autoencoder/#about-autoencoders","text":"","title":"About Autoencoders"},{"location":"deep_learning/practical_pytorch/pytorch_autoencoder/#feedforward-neural-network-fnn-to-autoencoders-aes","text":"Autoencoder is a form of unsupervised learning . This is a big deviation from what we have been doing: classification and regression which are under supervised learning. There are no labels required, inputs are used as labels. This is a bit mind-boggling for some, but there're many conrete use cases as you'll soon realize. Just a quick preview of use cases we will be covering: Denoising overcomplete AEs: recreate images without the random noises originally injected. Undercomplete AEs for anomaly detection: use AEs for credit card fraud detection via anomaly detection. Variational AEs for creating synthetic faces: with a convolutional VAEs, we can make fake faces. An autoencoder's purpose is to learn an approximation of the identity function (mapping \\(x\\) to \\(\\hat x\\) ). Essentially we are trying to learn a function that can take our input \\(x\\) and recreate it \\(\\hat x\\) . Technically we can do an exact recreation of our in-sample input if we use a very wide and deep neural network.","title":"Feedforward Neural Network (FNN) to Autoencoders (AEs)"},{"location":"deep_learning/practical_pytorch/pytorch_autoencoder/#undercomplete-and-overcomplete-autoencoders","text":"When we highlighted some use cases, did you notice how we mentioned undercomplete and autocomplete AEs? The only difference between the two is in the encoding output's size. In the diagram above, this refers to the encoding output's size after our first affine function (yellow box) and non-linear function (pink box). Undercomplete AEs: smaller This is when our encoding output's dimension is smaller than our input's dimension. Essentially we reduced the dimension of our data (dimensionality reduction) with an undercomplete AE Overcomplete AEs: larger This is when our encoding output's dimension is larger than our input's dimension Essentially we increased the dimension of our data with an overcomplete AE","title":"Undercomplete and Overcomplete Autoencoders"},{"location":"deep_learning/practical_pytorch/pytorch_autoencoder/#fully-connected-and-convolutional-autoencoders","text":"Another important point is that, in our diagram we've used the example of our Feedforward Neural Networks (FNN) where we use fully-connected layers. This is called Fully-connected AE. However, we can easily swap those fully-connected layers with convolutional layers. This is called Convolutional AE.","title":"Fully-connected and Convolutional Autoencoders"},{"location":"deep_learning/practical_pytorch/pytorch_autoencoder/#autoencoders-series","text":"We'll be covering a series of autoencoders in this order Fully-connected Overcomplete Autoencoder (AEs): Denoising Images Fully-connected Undercomplete Autoencoder (AEs): Credit Card Fraud Detection Convolutional Overcomplete Variational Autoencoder (VAEs): Generate Fake Human Faces Convolutional Overcomplete Adversarial Autoencoder (AAEs): Generate Fake Human Faces Generative Adversarial Networks (GANs): Generate Better Fake Human Faces","title":"Autoencoders Series"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/","text":"Convolutional Neural Network with PyTorch \u00b6 Run Jupyter Notebook You can run the code for this section in this jupyter notebook link . About Convolutional Neural Network \u00b6 Transition From Feedforward Neural Network \u00b6 Hidden Layer Feedforward Neural Network \u00b6 Recap of FNN So let's do a recap of what we covered in the Feedforward Neural Network (FNN) section using a simple FNN with 1 hidden layer (a pair of affine function and non-linear function) [Yellow box] Pass input into an affine function \\(\\boldsymbol{y} = A\\boldsymbol{x} + \\boldsymbol{b}\\) [Pink box] Pass logits to non-linear function, for example sigmoid, tanh (hyperbolic tangent), ReLU, or LeakyReLU [Blue box] Pass output of non-linear function to another affine function [Red box] Pass output of final affine function to softmax function to get our probability distribution over K classes [Purple box] Finally we can get our loss by using our cross entropy function Basic Convolutional Neural Network (CNN) \u00b6 A basic CNN just requires 2 additional layers! Convolution and pooling layers before our feedforward neural network Fully Connected (FC) Layer A layer with an affine function & non-linear function is called a Fully Connected (FC) layer One Convolutional Layer: High Level View \u00b6 One Convolutional Layer: High Level View Summary \u00b6 As the kernel is sliding/convolving across the image \\(\\rightarrow\\) 2 operations done per patch Element-wise multiplication Summation More kernels \\(=\\) more feature map channels Can capture more information about the input Multiple Convolutional Layers: High Level View \u00b6 Pooling Layer: High Level View \u00b6 2 Common Types Max Pooling Average Pooling Multiple Pooling Layers: High Level View \u00b6 Padding \u00b6 Padding Summary \u00b6 Valid Padding (No Padding) Output size < Input Size Same Padding (Zero Padding) Output size = Input Size Dimension Calculations \u00b6 \\(O = \\frac {W - K + 2P}{S} + 1\\) \\(O\\) : output height/length \\(W\\) : input height/length \\(K\\) : filter size (kernel size) \\(P\\) : padding \\(P = \\frac{K - 1}{2}\\) \\(S\\) : stride Example 1: Output Dimension Calculation for Valid Padding \u00b6 \\(W = 4\\) \\(K = 3\\) \\(P = 0\\) \\(S = 1\\) \\(O = \\frac {4 - 3 + 2*0}{1} + 1 = \\frac {1}{1} + 1 = 1 + 1 = 2\\) Example 2: Output Dimension Calculation for Same Padding \u00b6 \\(W = 5\\) \\(K = 3\\) \\(P = \\frac{3 - 1}{2} = \\frac{2}{2} = 1\\) \\(S = 1\\) \\(O = \\frac {5 - 3 + 2*1}{1} + 1 = \\frac {4}{1} + 1 = 5\\) Building a Convolutional Neural Network with PyTorch \u00b6 Model A: \u00b6 2 Convolutional Layers Same Padding (same output size) 2 Max Pooling Layers 1 Fully Connected Layer Steps \u00b6 Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model Step 1: Loading MNIST Train Dataset \u00b6 Images from 1 to 9 MNIST Dataset and Size of Training Dataset (Excluding Labels) import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) print ( train_dataset . train_data . size ()) torch.Size ([ 60000 , 28 , 28 ]) Size of our training dataset labels print ( train_dataset . train_labels . size ()) torch.Size ([ 60000 ]) Size of our testing dataset (excluding labels) print ( test_dataset . test_data . size ()) torch.Size ([ 10000 , 28 , 28 ]) Size of our testing dataset labels print ( test_dataset . test_labels . size ()) torch.Size ([ 10000 ]) Step 2: Make Dataset Iterable \u00b6 Load Dataset into Dataloader batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) Step 3: Create Model Class \u00b6 Output Formula for Convolution \u00b6 \\(O = \\frac {W - K + 2P}{S} + 1\\) \\(O\\) : output height/length \\(W\\) : input height/length \\(K\\) : filter size (kernel size) = 5 \\(P\\) : same padding (non-zero) \\(P = \\frac{K - 1}{2} = \\frac{5 - 1}{2} = 2\\) \\(S\\) : stride = 1 Output Formula for Pooling \u00b6 \\(O = \\frac {W - K}{S} + 1\\) W: input height/width K: filter size = 2 S: stride size = filter size , PyTorch defaults the stride to kernel filter size If using PyTorch default stride, this will result in the formula \\(O = \\frac {W}{K}\\) By default, in our tutorials, we do this for simplicity. Define our simple 2 convolutional layer CNN class CNNModel ( nn . Module ): def __init__ ( self ): super ( CNNModel , self ) . __init__ () # Convolution 1 self . cnn1 = nn . Conv2d ( in_channels = 1 , out_channels = 16 , kernel_size = 5 , stride = 1 , padding = 2 ) self . relu1 = nn . ReLU () # Max pool 1 self . maxpool1 = nn . MaxPool2d ( kernel_size = 2 ) # Convolution 2 self . cnn2 = nn . Conv2d ( in_channels = 16 , out_channels = 32 , kernel_size = 5 , stride = 1 , padding = 2 ) self . relu2 = nn . ReLU () # Max pool 2 self . maxpool2 = nn . MaxPool2d ( kernel_size = 2 ) # Fully connected 1 (readout) self . fc1 = nn . Linear ( 32 * 7 * 7 , 10 ) def forward ( self , x ): # Convolution 1 out = self . cnn1 ( x ) out = self . relu1 ( out ) # Max pool 1 out = self . maxpool1 ( out ) # Convolution 2 out = self . cnn2 ( out ) out = self . relu2 ( out ) # Max pool 2 out = self . maxpool2 ( out ) # Resize # Original size: (100, 32, 7, 7) # out.size(0): 100 # New out size: (100, 32*7*7) out = out . view ( out . size ( 0 ), - 1 ) # Linear function (readout) out = self . fc1 ( out ) return out Step 4: Instantiate Model Class \u00b6 Our model model = CNNModel () Step 5: Instantiate Loss Class \u00b6 Convolutional Neural Network: Cross Entropy Loss Feedforward Neural Network : Cross Entropy Loss Logistic Regression : Cross Entropy Loss Linear Regression : MSE Our cross entropy loss criterion = nn . CrossEntropyLoss () Step 6: Instantiate Optimizer Class \u00b6 Simplified equation \\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\) \\(\\theta\\) : parameters \\(\\eta\\) : learning rate (how fast we want to learn) \\(\\nabla_\\theta\\) : parameters' gradients Even simplier equation parameters = parameters - learning_rate * parameters_gradients At every iteration, we update our model's parameters Optimizer learning_rate = 0.01 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) Parameters In-Depth \u00b6 Print model's parameter print ( model . parameters ()) print ( len ( list ( model . parameters ()))) # Convolution 1: 16 Kernels print ( list ( model . parameters ())[ 0 ] . size ()) # Convolution 1 Bias: 16 Kernels print ( list ( model . parameters ())[ 1 ] . size ()) # Convolution 2: 32 Kernels with depth = 16 print ( list ( model . parameters ())[ 2 ] . size ()) # Convolution 2 Bias: 32 Kernels with depth = 16 print ( list ( model . parameters ())[ 3 ] . size ()) # Fully Connected Layer 1 print ( list ( model . parameters ())[ 4 ] . size ()) # Fully Connected Layer Bias print ( list ( model . parameters ())[ 5 ] . size ()) <generator object Module.parameters at 0x7f9864363c50> 6 torch.Size ([ 16 , 1 , 5 , 5 ]) torch.Size ([ 16 ]) torch.Size ([ 32 , 16 , 5 , 5 ]) torch.Size ([ 32 ]) torch.Size ([ 10 , 1568 ]) torch.Size ([ 10 ]) Step 7: Train Model \u00b6 Process Convert inputs to tensors with gradient accumulation abilities CNN Input: (1, 28, 28) Feedforward NN Input: (1, 28*28) Clear gradient buffets Get output given inputs Get loss Get gradients w.r.t. parameters Update parameters using gradients parameters = parameters - learning_rate * parameters_gradients REPEAT Model training iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images images = images . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images images = images . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500. Loss: 0.43324267864227295. Accuracy: 90 Iteration: 1000. Loss: 0.2511480152606964. Accuracy: 92 Iteration: 1500. Loss: 0.13431282341480255. Accuracy: 94 Iteration: 2000. Loss: 0.11173319816589355. Accuracy: 95 Iteration: 2500. Loss: 0.06409914791584015. Accuracy: 96 Iteration: 3000. Loss: 0.14377528429031372. Accuracy: 96 Model B: \u00b6 2 Convolutional Layers Same Padding (same output size) 2 Average Pooling Layers 1 Fully Connected Layer Steps \u00b6 Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 2 Conv + 2 Average Pool + 1 FC (Zero Padding, Same Padding) import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class CNNModel ( nn . Module ): def __init__ ( self ): super ( CNNModel , self ) . __init__ () # Convolution 1 self . cnn1 = nn . Conv2d ( in_channels = 1 , out_channels = 16 , kernel_size = 5 , stride = 1 , padding = 2 ) self . relu1 = nn . ReLU () # Average pool 1 self . avgpool1 = nn . AvgPool2d ( kernel_size = 2 ) # Convolution 2 self . cnn2 = nn . Conv2d ( in_channels = 16 , out_channels = 32 , kernel_size = 5 , stride = 1 , padding = 2 ) self . relu2 = nn . ReLU () # Average pool 2 self . avgpool2 = nn . AvgPool2d ( kernel_size = 2 ) # Fully connected 1 (readout) self . fc1 = nn . Linear ( 32 * 7 * 7 , 10 ) def forward ( self , x ): # Convolution 1 out = self . cnn1 ( x ) out = self . relu1 ( out ) # Average pool 1 out = self . avgpool1 ( out ) # Convolution 2 out = self . cnn2 ( out ) out = self . relu2 ( out ) # Max pool 2 out = self . avgpool2 ( out ) # Resize # Original size: (100, 32, 7, 7) # out.size(0): 100 # New out size: (100, 32*7*7) out = out . view ( out . size ( 0 ), - 1 ) # Linear function (readout) out = self . fc1 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' model = CNNModel () ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.01 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as tensors with gradient accumulation abilities images = images . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to tensors with gradient accumulation abilities images = images . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500 . Loss: 0 .6850348711013794. Accuracy: 85 Iteration: 1000 . Loss: 0 .36549052596092224. Accuracy: 88 Iteration: 1500 . Loss: 0 .31540098786354065. Accuracy: 89 Iteration: 2000 . Loss: 0 .3522164225578308. Accuracy: 90 Iteration: 2500 . Loss: 0 .2680729925632477. Accuracy: 91 Iteration: 3000 . Loss: 0 .26440390944480896. Accuracy: 92 Comparison of accuracies It seems like average pooling test accuracy is less than the max pooling accuracy! Does this mean average pooling is better? This is not definitive and depends on a lot of factors including the model's architecture, seed (that affects random weight initialization) and more. Model C: \u00b6 2 Convolutional Layers Valid Padding (smaller output size) 2 Max Pooling Layers 1 Fully Connected Layer Steps \u00b6 Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 2 Conv + 2 Max Pool + 1 FC (Valid Padding, No Padding) import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class CNNModel ( nn . Module ): def __init__ ( self ): super ( CNNModel , self ) . __init__ () # Convolution 1 self . cnn1 = nn . Conv2d ( in_channels = 1 , out_channels = 16 , kernel_size = 5 , stride = 1 , padding = 0 ) self . relu1 = nn . ReLU () # Max pool 1 self . maxpool1 = nn . MaxPool2d ( kernel_size = 2 ) # Convolution 2 self . cnn2 = nn . Conv2d ( in_channels = 16 , out_channels = 32 , kernel_size = 5 , stride = 1 , padding = 0 ) self . relu2 = nn . ReLU () # Max pool 2 self . maxpool2 = nn . MaxPool2d ( kernel_size = 2 ) # Fully connected 1 (readout) self . fc1 = nn . Linear ( 32 * 4 * 4 , 10 ) def forward ( self , x ): # Convolution 1 out = self . cnn1 ( x ) out = self . relu1 ( out ) # Max pool 1 out = self . maxpool1 ( out ) # Convolution 2 out = self . cnn2 ( out ) out = self . relu2 ( out ) # Max pool 2 out = self . maxpool2 ( out ) # Resize # Original size: (100, 32, 7, 7) # out.size(0): 100 # New out size: (100, 32*7*7) out = out . view ( out . size ( 0 ), - 1 ) # Linear function (readout) out = self . fc1 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' model = CNNModel () ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.01 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as tensors with gradient accumulation abilities images = images . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to tensors with gradient accumulation abilities images = images . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500 . Loss: 0 .5153220295906067. Accuracy: 88 Iteration: 1000 . Loss: 0 .28784745931625366. Accuracy: 92 Iteration: 1500 . Loss: 0 .4086027443408966. Accuracy: 94 Iteration: 2000 . Loss: 0 .09390712529420853. Accuracy: 95 Iteration: 2500 . Loss: 0 .07138358801603317. Accuracy: 95 Iteration: 3000 . Loss: 0 .05396252125501633. Accuracy: 96 Summary of Results \u00b6 Model A Model B Model C Max Pooling Average Pooling Max Pooling Same Padding Same Padding Valid Padding 97.04% 93.59% 96.5% All Models INPUT \\(\\rightarrow\\) CONV \\(\\rightarrow\\) POOL \\(\\rightarrow\\) CONV \\(\\rightarrow\\) POOL \\(\\rightarrow\\) FC Convolution Kernel Size = 5 x 5 Convolution Kernel Stride = 1 Pooling Kernel Size = 2 x 2 General Deep Learning Notes on CNN and FNN \u00b6 3 ways to expand a convolutional neural network More convolutional layers Less aggressive downsampling Smaller kernel size for pooling (gradually downsampling) More fully connected layers Cons Need a larger dataset Curse of dimensionality Does not necessarily mean higher accuracy 3. Building a Convolutional Neural Network with PyTorch (GPU) \u00b6 Model A \u00b6 GPU: 2 things must be on GPU - model - tensors Steps \u00b6 Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 2 Conv + 2 Max Pooling + 1 FC (Same Padding, Zero Padding) import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class CNNModel ( nn . Module ): def __init__ ( self ): super ( CNNModel , self ) . __init__ () # Convolution 1 self . cnn1 = nn . Conv2d ( in_channels = 1 , out_channels = 16 , kernel_size = 5 , stride = 1 , padding = 0 ) self . relu1 = nn . ReLU () # Max pool 1 self . maxpool1 = nn . MaxPool2d ( kernel_size = 2 ) # Convolution 2 self . cnn2 = nn . Conv2d ( in_channels = 16 , out_channels = 32 , kernel_size = 5 , stride = 1 , padding = 0 ) self . relu2 = nn . ReLU () # Max pool 2 self . maxpool2 = nn . MaxPool2d ( kernel_size = 2 ) # Fully connected 1 (readout) self . fc1 = nn . Linear ( 32 * 4 * 4 , 10 ) def forward ( self , x ): # Convolution 1 out = self . cnn1 ( x ) out = self . relu1 ( out ) # Max pool 1 out = self . maxpool1 ( out ) # Convolution 2 out = self . cnn2 ( out ) out = self . relu2 ( out ) # Max pool 2 out = self . maxpool2 ( out ) # Resize # Original size: (100, 32, 7, 7) # out.size(0): 100 # New out size: (100, 32*7*7) out = out . view ( out . size ( 0 ), - 1 ) # Linear function (readout) out = self . fc1 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' model = CNNModel () ####################### # USE GPU FOR MODEL # ####################### device = torch . device ( \"cuda:0\" if torch . cuda . is_available () else \"cpu\" ) model . to ( device ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.01 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): ####################### # USE GPU FOR MODEL # ####################### images = images . requires_grad_ () . to ( device ) labels = labels . to ( device ) # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : ####################### # USE GPU FOR MODEL # ####################### images = images . requires_grad_ () . to ( device ) labels = labels . to ( device ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) ####################### # USE GPU FOR MODEL # ####################### # Total correct predictions if torch . cuda . is_available (): correct += ( predicted . cpu () == labels . cpu ()) . sum () else : correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500 . Loss: 0 .36831170320510864. Accuracy: 88 Iteration: 1000 . Loss: 0 .31790846586227417. Accuracy: 92 Iteration: 1500 . Loss: 0 .1510857343673706. Accuracy: 94 Iteration: 2000 . Loss: 0 .08368007838726044. Accuracy: 95 Iteration: 2500 . Loss: 0 .13419771194458008. Accuracy: 96 Iteration: 3000 . Loss: 0 .16750787198543549. Accuracy: 96 More Efficient Convolutions via Toeplitz Matrices This is beyond the scope of this particular lesson. But now that we understand how convolutions work, it is critical to know that it is quite an inefficient operation if we use for-loops to perform our 2D convolutions (5 x 5 convolution kernel size for example) on our 2D images (28 x 28 MNIST image for example). A more efficient implementation is in converting our convolution kernel into a doubly block circulant/Toeplitz matrix (special case Toeplitz matrix) and our image (input) into a vector. Then, we will do just one matrix operation using our doulby block Toeplitz matrix and our input vector. There will be a whole lesson dedicated to this operation released down the road. Summary \u00b6 We've learnt to... Success Transition from Feedforward Neural Network Addition of Convolutional & Pooling Layers before Linear Layers One Convolutional Layer Basics One Pooling Layer Basics Max pooling Average pooling Padding Output Dimension Calculations and Examples \\(O = \\frac {W - K + 2P}{S} + 1\\) Convolutional Neural Networks Model A : 2 Conv + 2 Max pool + 1 FC Same Padding Model B : 2 Conv + 2 Average pool + 1 FC Same Padding Model C : 2 Conv + 2 Max pool + 1 FC Valid Padding Model Variation in Code Modifying only step 3 Ways to Expand Model\u2019s Capacity More convolutions Gradual pooling More fully connected layers GPU Code 2 things on GPU model tensors with gradient accumulation abilities Modifying only Step 4 & Step 7 7 Step Model Building Recap Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model Citation \u00b6 If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Convolutional Neural Networks (CNN)"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#convolutional-neural-network-with-pytorch","text":"Run Jupyter Notebook You can run the code for this section in this jupyter notebook link .","title":"Convolutional Neural Network with PyTorch"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#about-convolutional-neural-network","text":"","title":"About Convolutional Neural Network"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#transition-from-feedforward-neural-network","text":"","title":"Transition From Feedforward Neural Network"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#hidden-layer-feedforward-neural-network","text":"Recap of FNN So let's do a recap of what we covered in the Feedforward Neural Network (FNN) section using a simple FNN with 1 hidden layer (a pair of affine function and non-linear function) [Yellow box] Pass input into an affine function \\(\\boldsymbol{y} = A\\boldsymbol{x} + \\boldsymbol{b}\\) [Pink box] Pass logits to non-linear function, for example sigmoid, tanh (hyperbolic tangent), ReLU, or LeakyReLU [Blue box] Pass output of non-linear function to another affine function [Red box] Pass output of final affine function to softmax function to get our probability distribution over K classes [Purple box] Finally we can get our loss by using our cross entropy function","title":"Hidden Layer Feedforward Neural Network"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#basic-convolutional-neural-network-cnn","text":"A basic CNN just requires 2 additional layers! Convolution and pooling layers before our feedforward neural network Fully Connected (FC) Layer A layer with an affine function & non-linear function is called a Fully Connected (FC) layer","title":"Basic Convolutional Neural Network (CNN)"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#one-convolutional-layer-high-level-view","text":"","title":"One Convolutional Layer: High Level View"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#one-convolutional-layer-high-level-view-summary","text":"As the kernel is sliding/convolving across the image \\(\\rightarrow\\) 2 operations done per patch Element-wise multiplication Summation More kernels \\(=\\) more feature map channels Can capture more information about the input","title":"One Convolutional Layer: High Level View Summary"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#multiple-convolutional-layers-high-level-view","text":"","title":"Multiple Convolutional Layers: High Level View"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#pooling-layer-high-level-view","text":"2 Common Types Max Pooling Average Pooling","title":"Pooling Layer: High Level View"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#multiple-pooling-layers-high-level-view","text":"","title":"Multiple Pooling Layers: High Level View"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#padding","text":"","title":"Padding"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#padding-summary","text":"Valid Padding (No Padding) Output size < Input Size Same Padding (Zero Padding) Output size = Input Size","title":"Padding Summary"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#dimension-calculations","text":"\\(O = \\frac {W - K + 2P}{S} + 1\\) \\(O\\) : output height/length \\(W\\) : input height/length \\(K\\) : filter size (kernel size) \\(P\\) : padding \\(P = \\frac{K - 1}{2}\\) \\(S\\) : stride","title":"Dimension Calculations"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#example-1-output-dimension-calculation-for-valid-padding","text":"\\(W = 4\\) \\(K = 3\\) \\(P = 0\\) \\(S = 1\\) \\(O = \\frac {4 - 3 + 2*0}{1} + 1 = \\frac {1}{1} + 1 = 1 + 1 = 2\\)","title":"Example 1: Output Dimension Calculation for Valid Padding"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#example-2-output-dimension-calculation-for-same-padding","text":"\\(W = 5\\) \\(K = 3\\) \\(P = \\frac{3 - 1}{2} = \\frac{2}{2} = 1\\) \\(S = 1\\) \\(O = \\frac {5 - 3 + 2*1}{1} + 1 = \\frac {4}{1} + 1 = 5\\)","title":"Example 2: Output Dimension Calculation for Same Padding"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#building-a-convolutional-neural-network-with-pytorch","text":"","title":"Building a Convolutional Neural Network with PyTorch"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#model-a","text":"2 Convolutional Layers Same Padding (same output size) 2 Max Pooling Layers 1 Fully Connected Layer","title":"Model A:"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#steps","text":"Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model","title":"Steps"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#step-1-loading-mnist-train-dataset","text":"Images from 1 to 9 MNIST Dataset and Size of Training Dataset (Excluding Labels) import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) print ( train_dataset . train_data . size ()) torch.Size ([ 60000 , 28 , 28 ]) Size of our training dataset labels print ( train_dataset . train_labels . size ()) torch.Size ([ 60000 ]) Size of our testing dataset (excluding labels) print ( test_dataset . test_data . size ()) torch.Size ([ 10000 , 28 , 28 ]) Size of our testing dataset labels print ( test_dataset . test_labels . size ()) torch.Size ([ 10000 ])","title":"Step 1: Loading MNIST Train Dataset"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#step-2-make-dataset-iterable","text":"Load Dataset into Dataloader batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False )","title":"Step 2: Make Dataset Iterable"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#step-3-create-model-class","text":"","title":"Step 3: Create Model Class"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#output-formula-for-convolution","text":"\\(O = \\frac {W - K + 2P}{S} + 1\\) \\(O\\) : output height/length \\(W\\) : input height/length \\(K\\) : filter size (kernel size) = 5 \\(P\\) : same padding (non-zero) \\(P = \\frac{K - 1}{2} = \\frac{5 - 1}{2} = 2\\) \\(S\\) : stride = 1","title":"Output Formula for Convolution"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#output-formula-for-pooling","text":"\\(O = \\frac {W - K}{S} + 1\\) W: input height/width K: filter size = 2 S: stride size = filter size , PyTorch defaults the stride to kernel filter size If using PyTorch default stride, this will result in the formula \\(O = \\frac {W}{K}\\) By default, in our tutorials, we do this for simplicity. Define our simple 2 convolutional layer CNN class CNNModel ( nn . Module ): def __init__ ( self ): super ( CNNModel , self ) . __init__ () # Convolution 1 self . cnn1 = nn . Conv2d ( in_channels = 1 , out_channels = 16 , kernel_size = 5 , stride = 1 , padding = 2 ) self . relu1 = nn . ReLU () # Max pool 1 self . maxpool1 = nn . MaxPool2d ( kernel_size = 2 ) # Convolution 2 self . cnn2 = nn . Conv2d ( in_channels = 16 , out_channels = 32 , kernel_size = 5 , stride = 1 , padding = 2 ) self . relu2 = nn . ReLU () # Max pool 2 self . maxpool2 = nn . MaxPool2d ( kernel_size = 2 ) # Fully connected 1 (readout) self . fc1 = nn . Linear ( 32 * 7 * 7 , 10 ) def forward ( self , x ): # Convolution 1 out = self . cnn1 ( x ) out = self . relu1 ( out ) # Max pool 1 out = self . maxpool1 ( out ) # Convolution 2 out = self . cnn2 ( out ) out = self . relu2 ( out ) # Max pool 2 out = self . maxpool2 ( out ) # Resize # Original size: (100, 32, 7, 7) # out.size(0): 100 # New out size: (100, 32*7*7) out = out . view ( out . size ( 0 ), - 1 ) # Linear function (readout) out = self . fc1 ( out ) return out","title":"Output Formula for Pooling"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#step-4-instantiate-model-class","text":"Our model model = CNNModel ()","title":"Step 4: Instantiate Model Class"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#step-5-instantiate-loss-class","text":"Convolutional Neural Network: Cross Entropy Loss Feedforward Neural Network : Cross Entropy Loss Logistic Regression : Cross Entropy Loss Linear Regression : MSE Our cross entropy loss criterion = nn . CrossEntropyLoss ()","title":"Step 5: Instantiate Loss Class"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#step-6-instantiate-optimizer-class","text":"Simplified equation \\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\) \\(\\theta\\) : parameters \\(\\eta\\) : learning rate (how fast we want to learn) \\(\\nabla_\\theta\\) : parameters' gradients Even simplier equation parameters = parameters - learning_rate * parameters_gradients At every iteration, we update our model's parameters Optimizer learning_rate = 0.01 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate )","title":"Step 6: Instantiate Optimizer Class"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#parameters-in-depth","text":"Print model's parameter print ( model . parameters ()) print ( len ( list ( model . parameters ()))) # Convolution 1: 16 Kernels print ( list ( model . parameters ())[ 0 ] . size ()) # Convolution 1 Bias: 16 Kernels print ( list ( model . parameters ())[ 1 ] . size ()) # Convolution 2: 32 Kernels with depth = 16 print ( list ( model . parameters ())[ 2 ] . size ()) # Convolution 2 Bias: 32 Kernels with depth = 16 print ( list ( model . parameters ())[ 3 ] . size ()) # Fully Connected Layer 1 print ( list ( model . parameters ())[ 4 ] . size ()) # Fully Connected Layer Bias print ( list ( model . parameters ())[ 5 ] . size ()) <generator object Module.parameters at 0x7f9864363c50> 6 torch.Size ([ 16 , 1 , 5 , 5 ]) torch.Size ([ 16 ]) torch.Size ([ 32 , 16 , 5 , 5 ]) torch.Size ([ 32 ]) torch.Size ([ 10 , 1568 ]) torch.Size ([ 10 ])","title":"Parameters In-Depth"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#step-7-train-model","text":"Process Convert inputs to tensors with gradient accumulation abilities CNN Input: (1, 28, 28) Feedforward NN Input: (1, 28*28) Clear gradient buffets Get output given inputs Get loss Get gradients w.r.t. parameters Update parameters using gradients parameters = parameters - learning_rate * parameters_gradients REPEAT Model training iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images images = images . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images images = images . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500. Loss: 0.43324267864227295. Accuracy: 90 Iteration: 1000. Loss: 0.2511480152606964. Accuracy: 92 Iteration: 1500. Loss: 0.13431282341480255. Accuracy: 94 Iteration: 2000. Loss: 0.11173319816589355. Accuracy: 95 Iteration: 2500. Loss: 0.06409914791584015. Accuracy: 96 Iteration: 3000. Loss: 0.14377528429031372. Accuracy: 96","title":"Step 7: Train Model"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#model-b","text":"2 Convolutional Layers Same Padding (same output size) 2 Average Pooling Layers 1 Fully Connected Layer","title":"Model B:"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#steps_1","text":"Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 2 Conv + 2 Average Pool + 1 FC (Zero Padding, Same Padding) import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class CNNModel ( nn . Module ): def __init__ ( self ): super ( CNNModel , self ) . __init__ () # Convolution 1 self . cnn1 = nn . Conv2d ( in_channels = 1 , out_channels = 16 , kernel_size = 5 , stride = 1 , padding = 2 ) self . relu1 = nn . ReLU () # Average pool 1 self . avgpool1 = nn . AvgPool2d ( kernel_size = 2 ) # Convolution 2 self . cnn2 = nn . Conv2d ( in_channels = 16 , out_channels = 32 , kernel_size = 5 , stride = 1 , padding = 2 ) self . relu2 = nn . ReLU () # Average pool 2 self . avgpool2 = nn . AvgPool2d ( kernel_size = 2 ) # Fully connected 1 (readout) self . fc1 = nn . Linear ( 32 * 7 * 7 , 10 ) def forward ( self , x ): # Convolution 1 out = self . cnn1 ( x ) out = self . relu1 ( out ) # Average pool 1 out = self . avgpool1 ( out ) # Convolution 2 out = self . cnn2 ( out ) out = self . relu2 ( out ) # Max pool 2 out = self . avgpool2 ( out ) # Resize # Original size: (100, 32, 7, 7) # out.size(0): 100 # New out size: (100, 32*7*7) out = out . view ( out . size ( 0 ), - 1 ) # Linear function (readout) out = self . fc1 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' model = CNNModel () ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.01 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as tensors with gradient accumulation abilities images = images . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to tensors with gradient accumulation abilities images = images . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500 . Loss: 0 .6850348711013794. Accuracy: 85 Iteration: 1000 . Loss: 0 .36549052596092224. Accuracy: 88 Iteration: 1500 . Loss: 0 .31540098786354065. Accuracy: 89 Iteration: 2000 . Loss: 0 .3522164225578308. Accuracy: 90 Iteration: 2500 . Loss: 0 .2680729925632477. Accuracy: 91 Iteration: 3000 . Loss: 0 .26440390944480896. Accuracy: 92 Comparison of accuracies It seems like average pooling test accuracy is less than the max pooling accuracy! Does this mean average pooling is better? This is not definitive and depends on a lot of factors including the model's architecture, seed (that affects random weight initialization) and more.","title":"Steps"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#model-c","text":"2 Convolutional Layers Valid Padding (smaller output size) 2 Max Pooling Layers 1 Fully Connected Layer","title":"Model C:"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#steps_2","text":"Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 2 Conv + 2 Max Pool + 1 FC (Valid Padding, No Padding) import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class CNNModel ( nn . Module ): def __init__ ( self ): super ( CNNModel , self ) . __init__ () # Convolution 1 self . cnn1 = nn . Conv2d ( in_channels = 1 , out_channels = 16 , kernel_size = 5 , stride = 1 , padding = 0 ) self . relu1 = nn . ReLU () # Max pool 1 self . maxpool1 = nn . MaxPool2d ( kernel_size = 2 ) # Convolution 2 self . cnn2 = nn . Conv2d ( in_channels = 16 , out_channels = 32 , kernel_size = 5 , stride = 1 , padding = 0 ) self . relu2 = nn . ReLU () # Max pool 2 self . maxpool2 = nn . MaxPool2d ( kernel_size = 2 ) # Fully connected 1 (readout) self . fc1 = nn . Linear ( 32 * 4 * 4 , 10 ) def forward ( self , x ): # Convolution 1 out = self . cnn1 ( x ) out = self . relu1 ( out ) # Max pool 1 out = self . maxpool1 ( out ) # Convolution 2 out = self . cnn2 ( out ) out = self . relu2 ( out ) # Max pool 2 out = self . maxpool2 ( out ) # Resize # Original size: (100, 32, 7, 7) # out.size(0): 100 # New out size: (100, 32*7*7) out = out . view ( out . size ( 0 ), - 1 ) # Linear function (readout) out = self . fc1 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' model = CNNModel () ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.01 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as tensors with gradient accumulation abilities images = images . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to tensors with gradient accumulation abilities images = images . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500 . Loss: 0 .5153220295906067. Accuracy: 88 Iteration: 1000 . Loss: 0 .28784745931625366. Accuracy: 92 Iteration: 1500 . Loss: 0 .4086027443408966. Accuracy: 94 Iteration: 2000 . Loss: 0 .09390712529420853. Accuracy: 95 Iteration: 2500 . Loss: 0 .07138358801603317. Accuracy: 95 Iteration: 3000 . Loss: 0 .05396252125501633. Accuracy: 96","title":"Steps"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#summary-of-results","text":"Model A Model B Model C Max Pooling Average Pooling Max Pooling Same Padding Same Padding Valid Padding 97.04% 93.59% 96.5% All Models INPUT \\(\\rightarrow\\) CONV \\(\\rightarrow\\) POOL \\(\\rightarrow\\) CONV \\(\\rightarrow\\) POOL \\(\\rightarrow\\) FC Convolution Kernel Size = 5 x 5 Convolution Kernel Stride = 1 Pooling Kernel Size = 2 x 2","title":"Summary of Results"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#general-deep-learning-notes-on-cnn-and-fnn","text":"3 ways to expand a convolutional neural network More convolutional layers Less aggressive downsampling Smaller kernel size for pooling (gradually downsampling) More fully connected layers Cons Need a larger dataset Curse of dimensionality Does not necessarily mean higher accuracy","title":"General Deep Learning Notes on CNN and FNN"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#3-building-a-convolutional-neural-network-with-pytorch-gpu","text":"","title":"3. Building a Convolutional Neural Network with PyTorch (GPU)"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#model-a_1","text":"GPU: 2 things must be on GPU - model - tensors","title":"Model A"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#steps_3","text":"Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 2 Conv + 2 Max Pooling + 1 FC (Same Padding, Zero Padding) import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class CNNModel ( nn . Module ): def __init__ ( self ): super ( CNNModel , self ) . __init__ () # Convolution 1 self . cnn1 = nn . Conv2d ( in_channels = 1 , out_channels = 16 , kernel_size = 5 , stride = 1 , padding = 0 ) self . relu1 = nn . ReLU () # Max pool 1 self . maxpool1 = nn . MaxPool2d ( kernel_size = 2 ) # Convolution 2 self . cnn2 = nn . Conv2d ( in_channels = 16 , out_channels = 32 , kernel_size = 5 , stride = 1 , padding = 0 ) self . relu2 = nn . ReLU () # Max pool 2 self . maxpool2 = nn . MaxPool2d ( kernel_size = 2 ) # Fully connected 1 (readout) self . fc1 = nn . Linear ( 32 * 4 * 4 , 10 ) def forward ( self , x ): # Convolution 1 out = self . cnn1 ( x ) out = self . relu1 ( out ) # Max pool 1 out = self . maxpool1 ( out ) # Convolution 2 out = self . cnn2 ( out ) out = self . relu2 ( out ) # Max pool 2 out = self . maxpool2 ( out ) # Resize # Original size: (100, 32, 7, 7) # out.size(0): 100 # New out size: (100, 32*7*7) out = out . view ( out . size ( 0 ), - 1 ) # Linear function (readout) out = self . fc1 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' model = CNNModel () ####################### # USE GPU FOR MODEL # ####################### device = torch . device ( \"cuda:0\" if torch . cuda . is_available () else \"cpu\" ) model . to ( device ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.01 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): ####################### # USE GPU FOR MODEL # ####################### images = images . requires_grad_ () . to ( device ) labels = labels . to ( device ) # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : ####################### # USE GPU FOR MODEL # ####################### images = images . requires_grad_ () . to ( device ) labels = labels . to ( device ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) ####################### # USE GPU FOR MODEL # ####################### # Total correct predictions if torch . cuda . is_available (): correct += ( predicted . cpu () == labels . cpu ()) . sum () else : correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration: 500 . Loss: 0 .36831170320510864. Accuracy: 88 Iteration: 1000 . Loss: 0 .31790846586227417. Accuracy: 92 Iteration: 1500 . Loss: 0 .1510857343673706. Accuracy: 94 Iteration: 2000 . Loss: 0 .08368007838726044. Accuracy: 95 Iteration: 2500 . Loss: 0 .13419771194458008. Accuracy: 96 Iteration: 3000 . Loss: 0 .16750787198543549. Accuracy: 96 More Efficient Convolutions via Toeplitz Matrices This is beyond the scope of this particular lesson. But now that we understand how convolutions work, it is critical to know that it is quite an inefficient operation if we use for-loops to perform our 2D convolutions (5 x 5 convolution kernel size for example) on our 2D images (28 x 28 MNIST image for example). A more efficient implementation is in converting our convolution kernel into a doubly block circulant/Toeplitz matrix (special case Toeplitz matrix) and our image (input) into a vector. Then, we will do just one matrix operation using our doulby block Toeplitz matrix and our input vector. There will be a whole lesson dedicated to this operation released down the road.","title":"Steps"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#summary","text":"We've learnt to... Success Transition from Feedforward Neural Network Addition of Convolutional & Pooling Layers before Linear Layers One Convolutional Layer Basics One Pooling Layer Basics Max pooling Average pooling Padding Output Dimension Calculations and Examples \\(O = \\frac {W - K + 2P}{S} + 1\\) Convolutional Neural Networks Model A : 2 Conv + 2 Max pool + 1 FC Same Padding Model B : 2 Conv + 2 Average pool + 1 FC Same Padding Model C : 2 Conv + 2 Max pool + 1 FC Valid Padding Model Variation in Code Modifying only step 3 Ways to Expand Model\u2019s Capacity More convolutions Gradual pooling More fully connected layers GPU Code 2 things on GPU model tensors with gradient accumulation abilities Modifying only Step 4 & Step 7 7 Step Model Building Recap Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model","title":"Summary"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#citation","text":"If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Citation"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/","text":"Overcomplete Autoencoders with PyTorch \u00b6 Run Jupyter Notebook You can run the code for this section in this jupyter notebook link . Introduction \u00b6 Recap! An autoencoder's purpose is to learn an approximation of the identity function (mapping \\(x\\) to \\(\\hat x\\) ). Essentially we are trying to learn a function that can take our input \\(x\\) and recreate it \\(\\hat x\\) . Technically we can do an exact recreation of our in-sample input if we use a very wide and deep neural network. In this particular tutorial, we will be covering denoising autoencoder through overcomplete encoders. Essentially given noisy images, you can denoise and make them less noisy with this tutorial through overcomplete encoders. Fashion MNIST Dataset Exploration \u00b6 Imports \u00b6 import torch import torch.nn as nn import torch.nn.functional as F import torchvision.transforms as transforms import torchvision.datasets as dsets import matplotlib.pyplot as plt import numpy as np % matplotlib inline figsize = ( 15 , 6 ) plt . style . use ( 'fivethirtyeight' ) Name Content Examples Size Link MD5 Checksum train-images-idx3-ubyte.gz training set images 60,000 26 MBytes Download 8d4fb7e6c68d591d4c3dfef9ec88bf0d train-labels-idx1-ubyte.gz training set labels 60,000 29 KBytes Download 25c81989df183df01b3e8a0aad5dffbe t10k-images-idx3-ubyte.gz test set images 10,000 4.3 MBytes Download bef4ecab320f06d8554ea6380940ec79 t10k-labels-idx1-ubyte.gz test set labels 10,000 5.1 KBytes Download bb300cfdad3c16e7a12a480ee83cd310 Load Dataset (Step 1) \u00b6 Typically we've been leveraging on dsets.MNIST() , now we simply change to dsets.FashionMNIST ! # Fashion-MNIST data loader train_dataset = dsets . FashionMNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . FashionMNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz Processing... Done! Load Data Loader (Step 2) \u00b6 # Batch size, iterations and epochs batch_size = 100 n_iters = 5000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) Labels \u00b6 Each training and test example is assigned to one of the following labels: Label Description 0 T-shirt/top 1 Trouser 2 Pullover 3 Dress 4 Coat 5 Sandal 6 Shirt 7 Sneaker 8 Bag 9 Ankle boot Sample: Boot \u00b6 # Sample 0: boot sample_num = 0 show_img = train_dataset [ sample_num ][ 0 ] . numpy () . reshape ( 28 , 28 ) label = train_dataset [ sample_num ][ 1 ] print ( f 'Label { label } ' ) plt . imshow ( show_img , cmap = 'gray' ); Label 9 Sample: shirt \u00b6 # Sample 1: shirt sample_num = 1 show_img = train_dataset [ sample_num ][ 0 ] . numpy () . reshape ( 28 , 28 ) label = train_dataset [ sample_num ][ 1 ] print ( f 'Label { label } ' ) plt . imshow ( show_img , cmap = 'gray' ); Label 0 Sample: dress \u00b6 # Sample 3: dress sample_num = 3 show_img = train_dataset [ sample_num ][ 0 ] . numpy () . reshape ( 28 , 28 ) label = train_dataset [ sample_num ][ 1 ] print ( f 'Label { label } ' ) plt . imshow ( show_img , cmap = 'gray' ); Label 3 Maximum/minimum pixel values \u00b6 Pixel values range from 0 to 1 min_pixel_value = train_dataset [ sample_num ][ 0 ] . min () max_pixel_value = train_dataset [ sample_num ][ 0 ] . max () print ( f 'Minimum pixel value: { min_pixel_value } ' ) print ( f 'Maximum pixel value: { max_pixel_value } ' ) Minimum pixel value: 0.0 Maximum pixel value: 1.0 Overcomplete Autoencoder \u00b6 Sigmoid Function \u00b6 Sigmoid function was introduced earlier, where the function allows to bound our output from 0 to 1 inclusive given our input. This is introduced and clarified here as we would want this in our final layer of our overcomplete autoencoder as we want to bound out final output to the pixels' range of 0 and 1. # Sigmoid function has function bounded by min=0 and max=1 # So this will be what we will be using for the final layer's function x = torch . arange ( - 10. , 10. , 0.1 ) plt . figure ( figsize = figsize ); plt . plot ( x . numpy (), torch . sigmoid ( x ) . numpy ()) plt . title ( 'Sigmoid Function' ) Text(0.5, 1.0, 'Sigmoid Function') Steps \u00b6 These steps should be familiar by now! Our famous 7 steps. But I will be adding one more step here, Step 8 where we run our inference. Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model Step 8: Model Inference Step 3: Create Model Class \u00b6 # Model definition class FullyConnectedAutoencoder ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super () . __init__ () # Encoder: affine function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Decoder: affine function self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Encoder: affine function out = self . fc1 ( x ) # Encoder: non-linear function out = F . leaky_relu ( out ) # Decoder: affine function out = self . fc2 ( out ) # Decoder: non-linear function out = torch . sigmoid ( out ) return out Step 4: Instantiate Model Class \u00b6 # Dimensions for overcomplete (larger latent representation) input_dim = 28 * 28 hidden_dim = int ( input_dim * 1.5 ) output_dim = input_dim # Instantiate Fully-connected Autoencoder (FC-AE) # And assign to model object model = FullyConnectedAutoencoder ( input_dim , hidden_dim , output_dim ) Step 5: Instantiate Loss Class \u00b6 # We want to minimize the per pixel reconstruction loss # So we've to use the mean squared error (MSE) loss # This is similar to our regression tasks' loss criterion = nn . MSELoss () Step 6: Instantiate Optimizer Class \u00b6 # Using basic Adam optimizer learning_rate = 1e-3 optimizer = torch . optim . Adam ( model . parameters (), lr = learning_rate ) Inspect Parameter Groups \u00b6 # Parameter inspection num_params_group = len ( list ( model . parameters ())) for group_idx in range ( num_params_group ): print ( list ( model . parameters ())[ group_idx ] . size ()) torch.Size([1176, 784]) torch.Size([1176]) torch.Size([784, 1176]) torch.Size([784]) Step 7: Train Model \u00b6 Take note there's a critical line of dropout = nn.Dropout(0.5) here which basically allows us to make noisy images from our original Fashion MNIST images. It basically drops out 50% of all pixels randomly. idx = 0 # Dropout for creating noisy images # by dropping out pixel with a 50% probability dropout = nn . Dropout ( 0.5 ) for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Noisy images noisy_images = dropout ( torch . ones ( images . shape )) * images # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output outputs = model ( noisy_images ) # Calculate Loss: MSE Loss based on pixel-to-pixel comparison loss = criterion ( outputs , images ) # Getting gradients w.r.t. parameters via backpropagation loss . backward () # Updating parameters via gradient descent optimizer . step () idx += 1 if idx % 500 == 0 : # Calculate MSE Test Loss total_test_loss = 0 total_samples = 0 # Iterate through test dataset for images , labels in test_loader : # Noisy images noisy_images = dropout ( torch . ones ( images . shape )) * images # Forward pass only to get logits/output outputs = model ( noisy_images . view ( - 1 , 28 * 28 )) # Test loss test_loss = criterion ( outputs , images . view ( - 1 , 28 * 28 )) # Total number of labels total_samples += labels . size ( 0 ) # Total test loss total_test_loss += test_loss mean_test_loss = total_test_loss / total_samples # Print Loss print ( f 'Iteration: { idx } . Average Test Loss: { mean_test_loss . item () } .' ) Iteration: 500. Average Test Loss: 0.0001664187147980556. Iteration: 1000. Average Test Loss: 0.00014121478307060897. Iteration: 1500. Average Test Loss: 0.0001341506722383201. Iteration: 2000. Average Test Loss: 0.0001252180663868785. Iteration: 2500. Average Test Loss: 0.00012206179235363379. Iteration: 3000. Average Test Loss: 0.00011766648094635457. Iteration: 3500. Average Test Loss: 0.00011584569438127801. Iteration: 4000. Average Test Loss: 0.00011396891932236031. Iteration: 4500. Average Test Loss: 0.00011224475019844249. Model Inference \u00b6 Raw Sample 1 \u00b6 # Test sample: Raw sample_num = 10 raw_img = test_dataset [ sample_num ][ 0 ] show_img = raw_img . numpy () . reshape ( 28 , 28 ) label = test_dataset [ sample_num ][ 1 ] print ( f 'Label { label } ' ) plt . imshow ( show_img , cmap = 'gray' ); Label 4 Test Sample 1: Noisy to Denoised \u00b6 # Test sample: Noisy sample_num = 10 raw_img = test_dataset [ sample_num ][ 0 ] noisy_image = dropout ( torch . ones ( raw_img . shape )) * raw_img show_img = noisy_image . numpy () . reshape ( 28 , 28 ) label = test_dataset [ sample_num ][ 1 ] print ( f 'Label { label } ' ) plt . imshow ( show_img , cmap = 'gray' ); Label 4 Summary \u00b6 Introduction Recap of Autoencoders Introduction of denoising autoencoders Dataset Exploration Fashion MNIST 10 classes Similar to MNIST but fashion images instead of digits 8 Step Model Building Recap Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model Step 8: Model Inference Raw Sample 1 Noisy to Denoised Sample 1","title":"Fully-connected Overcomplete Autoencoder (AE)"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#overcomplete-autoencoders-with-pytorch","text":"Run Jupyter Notebook You can run the code for this section in this jupyter notebook link .","title":"Overcomplete Autoencoders with PyTorch"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#introduction","text":"Recap! An autoencoder's purpose is to learn an approximation of the identity function (mapping \\(x\\) to \\(\\hat x\\) ). Essentially we are trying to learn a function that can take our input \\(x\\) and recreate it \\(\\hat x\\) . Technically we can do an exact recreation of our in-sample input if we use a very wide and deep neural network. In this particular tutorial, we will be covering denoising autoencoder through overcomplete encoders. Essentially given noisy images, you can denoise and make them less noisy with this tutorial through overcomplete encoders.","title":"Introduction"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#fashion-mnist-dataset-exploration","text":"","title":"Fashion MNIST Dataset Exploration"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#imports","text":"import torch import torch.nn as nn import torch.nn.functional as F import torchvision.transforms as transforms import torchvision.datasets as dsets import matplotlib.pyplot as plt import numpy as np % matplotlib inline figsize = ( 15 , 6 ) plt . style . use ( 'fivethirtyeight' ) Name Content Examples Size Link MD5 Checksum train-images-idx3-ubyte.gz training set images 60,000 26 MBytes Download 8d4fb7e6c68d591d4c3dfef9ec88bf0d train-labels-idx1-ubyte.gz training set labels 60,000 29 KBytes Download 25c81989df183df01b3e8a0aad5dffbe t10k-images-idx3-ubyte.gz test set images 10,000 4.3 MBytes Download bef4ecab320f06d8554ea6380940ec79 t10k-labels-idx1-ubyte.gz test set labels 10,000 5.1 KBytes Download bb300cfdad3c16e7a12a480ee83cd310","title":"Imports"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#load-dataset-step-1","text":"Typically we've been leveraging on dsets.MNIST() , now we simply change to dsets.FashionMNIST ! # Fashion-MNIST data loader train_dataset = dsets . FashionMNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . FashionMNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz Processing... Done!","title":"Load Dataset (Step 1)"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#load-data-loader-step-2","text":"# Batch size, iterations and epochs batch_size = 100 n_iters = 5000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False )","title":"Load Data Loader (Step 2)"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#labels","text":"Each training and test example is assigned to one of the following labels: Label Description 0 T-shirt/top 1 Trouser 2 Pullover 3 Dress 4 Coat 5 Sandal 6 Shirt 7 Sneaker 8 Bag 9 Ankle boot","title":"Labels"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#sample-boot","text":"# Sample 0: boot sample_num = 0 show_img = train_dataset [ sample_num ][ 0 ] . numpy () . reshape ( 28 , 28 ) label = train_dataset [ sample_num ][ 1 ] print ( f 'Label { label } ' ) plt . imshow ( show_img , cmap = 'gray' ); Label 9","title":"Sample: Boot"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#sample-shirt","text":"# Sample 1: shirt sample_num = 1 show_img = train_dataset [ sample_num ][ 0 ] . numpy () . reshape ( 28 , 28 ) label = train_dataset [ sample_num ][ 1 ] print ( f 'Label { label } ' ) plt . imshow ( show_img , cmap = 'gray' ); Label 0","title":"Sample: shirt"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#sample-dress","text":"# Sample 3: dress sample_num = 3 show_img = train_dataset [ sample_num ][ 0 ] . numpy () . reshape ( 28 , 28 ) label = train_dataset [ sample_num ][ 1 ] print ( f 'Label { label } ' ) plt . imshow ( show_img , cmap = 'gray' ); Label 3","title":"Sample: dress"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#maximumminimum-pixel-values","text":"Pixel values range from 0 to 1 min_pixel_value = train_dataset [ sample_num ][ 0 ] . min () max_pixel_value = train_dataset [ sample_num ][ 0 ] . max () print ( f 'Minimum pixel value: { min_pixel_value } ' ) print ( f 'Maximum pixel value: { max_pixel_value } ' ) Minimum pixel value: 0.0 Maximum pixel value: 1.0","title":"Maximum/minimum pixel values"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#overcomplete-autoencoder","text":"","title":"Overcomplete Autoencoder"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#sigmoid-function","text":"Sigmoid function was introduced earlier, where the function allows to bound our output from 0 to 1 inclusive given our input. This is introduced and clarified here as we would want this in our final layer of our overcomplete autoencoder as we want to bound out final output to the pixels' range of 0 and 1. # Sigmoid function has function bounded by min=0 and max=1 # So this will be what we will be using for the final layer's function x = torch . arange ( - 10. , 10. , 0.1 ) plt . figure ( figsize = figsize ); plt . plot ( x . numpy (), torch . sigmoid ( x ) . numpy ()) plt . title ( 'Sigmoid Function' ) Text(0.5, 1.0, 'Sigmoid Function')","title":"Sigmoid Function"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#steps","text":"These steps should be familiar by now! Our famous 7 steps. But I will be adding one more step here, Step 8 where we run our inference. Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model Step 8: Model Inference","title":"Steps"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#step-3-create-model-class","text":"# Model definition class FullyConnectedAutoencoder ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super () . __init__ () # Encoder: affine function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Decoder: affine function self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Encoder: affine function out = self . fc1 ( x ) # Encoder: non-linear function out = F . leaky_relu ( out ) # Decoder: affine function out = self . fc2 ( out ) # Decoder: non-linear function out = torch . sigmoid ( out ) return out","title":"Step 3: Create Model Class"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#step-4-instantiate-model-class","text":"# Dimensions for overcomplete (larger latent representation) input_dim = 28 * 28 hidden_dim = int ( input_dim * 1.5 ) output_dim = input_dim # Instantiate Fully-connected Autoencoder (FC-AE) # And assign to model object model = FullyConnectedAutoencoder ( input_dim , hidden_dim , output_dim )","title":"Step 4: Instantiate Model Class"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#step-5-instantiate-loss-class","text":"# We want to minimize the per pixel reconstruction loss # So we've to use the mean squared error (MSE) loss # This is similar to our regression tasks' loss criterion = nn . MSELoss ()","title":"Step 5: Instantiate Loss Class"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#step-6-instantiate-optimizer-class","text":"# Using basic Adam optimizer learning_rate = 1e-3 optimizer = torch . optim . Adam ( model . parameters (), lr = learning_rate )","title":"Step 6: Instantiate Optimizer Class"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#inspect-parameter-groups","text":"# Parameter inspection num_params_group = len ( list ( model . parameters ())) for group_idx in range ( num_params_group ): print ( list ( model . parameters ())[ group_idx ] . size ()) torch.Size([1176, 784]) torch.Size([1176]) torch.Size([784, 1176]) torch.Size([784])","title":"Inspect Parameter Groups"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#step-7-train-model","text":"Take note there's a critical line of dropout = nn.Dropout(0.5) here which basically allows us to make noisy images from our original Fashion MNIST images. It basically drops out 50% of all pixels randomly. idx = 0 # Dropout for creating noisy images # by dropping out pixel with a 50% probability dropout = nn . Dropout ( 0.5 ) for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Noisy images noisy_images = dropout ( torch . ones ( images . shape )) * images # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output outputs = model ( noisy_images ) # Calculate Loss: MSE Loss based on pixel-to-pixel comparison loss = criterion ( outputs , images ) # Getting gradients w.r.t. parameters via backpropagation loss . backward () # Updating parameters via gradient descent optimizer . step () idx += 1 if idx % 500 == 0 : # Calculate MSE Test Loss total_test_loss = 0 total_samples = 0 # Iterate through test dataset for images , labels in test_loader : # Noisy images noisy_images = dropout ( torch . ones ( images . shape )) * images # Forward pass only to get logits/output outputs = model ( noisy_images . view ( - 1 , 28 * 28 )) # Test loss test_loss = criterion ( outputs , images . view ( - 1 , 28 * 28 )) # Total number of labels total_samples += labels . size ( 0 ) # Total test loss total_test_loss += test_loss mean_test_loss = total_test_loss / total_samples # Print Loss print ( f 'Iteration: { idx } . Average Test Loss: { mean_test_loss . item () } .' ) Iteration: 500. Average Test Loss: 0.0001664187147980556. Iteration: 1000. Average Test Loss: 0.00014121478307060897. Iteration: 1500. Average Test Loss: 0.0001341506722383201. Iteration: 2000. Average Test Loss: 0.0001252180663868785. Iteration: 2500. Average Test Loss: 0.00012206179235363379. Iteration: 3000. Average Test Loss: 0.00011766648094635457. Iteration: 3500. Average Test Loss: 0.00011584569438127801. Iteration: 4000. Average Test Loss: 0.00011396891932236031. Iteration: 4500. Average Test Loss: 0.00011224475019844249.","title":"Step 7: Train Model"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#model-inference","text":"","title":"Model Inference"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#raw-sample-1","text":"# Test sample: Raw sample_num = 10 raw_img = test_dataset [ sample_num ][ 0 ] show_img = raw_img . numpy () . reshape ( 28 , 28 ) label = test_dataset [ sample_num ][ 1 ] print ( f 'Label { label } ' ) plt . imshow ( show_img , cmap = 'gray' ); Label 4","title":"Raw Sample 1"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#test-sample-1-noisy-to-denoised","text":"# Test sample: Noisy sample_num = 10 raw_img = test_dataset [ sample_num ][ 0 ] noisy_image = dropout ( torch . ones ( raw_img . shape )) * raw_img show_img = noisy_image . numpy () . reshape ( 28 , 28 ) label = test_dataset [ sample_num ][ 1 ] print ( f 'Label { label } ' ) plt . imshow ( show_img , cmap = 'gray' ); Label 4","title":"Test Sample 1: Noisy to Denoised"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#summary","text":"Introduction Recap of Autoencoders Introduction of denoising autoencoders Dataset Exploration Fashion MNIST 10 classes Similar to MNIST but fashion images instead of digits 8 Step Model Building Recap Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model Step 8: Model Inference Raw Sample 1 Noisy to Denoised Sample 1","title":"Summary"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/","text":"Feedforward Neural Network with PyTorch \u00b6 Run Jupyter Notebook You can run the code for this section in this jupyter notebook link . About Feedforward Neural Network \u00b6 Logistic Regression Transition to Neural Networks \u00b6 Logistic Regression Review \u00b6 Define logistic regression model Import our relevant torch modules. import torch import torch.nn as nn Define our model class. class LogisticRegressionModel ( nn . Module ): def __init__ ( self , input_dim , output_dim ): super ( LogisticRegressionModel , self ) . __init__ () self . linear = nn . Linear ( input_dim , output_dim ) def forward ( self , x ): out = self . linear ( x ) return out Instantiate the logistic regression model. input_dim = 28 * 28 output_dim = 10 model = LogisticRegressionModel ( input_dim , output_dim ) When we inspect the model, we would have an input size of 784 (derived from 28 x 28) and output size of 10 (which is the number of classes we are classifying from 0 to 9). print ( model ) LogisticRegressionModel ( ( linear ): Linear ( in_features = 784 , out_features = 10 , bias = True ) ) Logistic Regression Problems \u00b6 Can represent linear functions well \\(y = 2x + 3\\) \\(y = x_1 + x_2\\) \\(y = x_1 + 3x_2 + 4x_3\\) Cannot represent non-linear functions \\(y = 4x_1 + 2x_2^2 +3x_3^3\\) \\(y = x_1x_2\\) Introducing a Non-linear Function \u00b6 Non-linear Function In-Depth \u00b6 Function: takes a number & perform mathematical operation Common Types of Non-linearity ReLUs (Rectified Linear Units) Sigmoid Tanh Sigmoid (Logistic) \u00b6 \\(\\sigma(x) = \\frac{1}{1 + e^{-x}}\\) Input number \\(\\rightarrow\\) [0, 1] Large negative number \\(\\rightarrow\\) 0 Large positive number \\(\\rightarrow\\) 1 Cons: Activation saturates at 0 or 1 with gradients \\(\\approx\\) 0 No signal to update weights \\(\\rightarrow\\) cannot learn Solution: Have to carefully initialize weights to prevent this Outputs not centered around 0 If output always positive \\(\\rightarrow\\) gradients always positive or negative \\(\\rightarrow\\) bad for gradient updates Tanh \u00b6 \\(\\tanh(x) = 2 \\sigma(2x) -1\\) A scaled sigmoid function Input number \\(\\rightarrow\\) [-1, 1] Cons: Activation saturates at 0 or 1 with gradients \\(\\approx\\) 0 No signal to update weights \\(\\rightarrow\\) cannot learn Solution : Have to carefully initialize weights to prevent this ReLUs \u00b6 \\(f(x) = \\max(0, x)\\) Pros: Accelerates convergence \\(\\rightarrow\\) train faster Less computationally expensive operation compared to Sigmoid/Tanh exponentials Cons: Many ReLU units \"die\" \\(\\rightarrow\\) gradients = 0 forever Solution : careful learning rate choice Building a Feedforward Neural Network with PyTorch \u00b6 Model A: 1 Hidden Layer Feedforward Neural Network (Sigmoid Activation) \u00b6 Steps \u00b6 Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model Step 1: Loading MNIST Train Dataset \u00b6 Images from 1 to 9 Similar to what we did in logistic regression, we will be using the same MNIST dataset where we load our training and testing datasets. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) Step 2: Make Dataset Iterable \u00b6 Batch sizes and iterations Because we have 60000 training samples (images), we need to split them up to small groups (batches) and pass these batches of samples to our feedforward neural network subsesquently. There are a few reasons why we split them into batches. Passing your whole dataset as a single batch would: (1) require a lot of RAM/VRAM on your CPU/GPU and this might result in Out-of-Memory (OOM) errors. (2) cause unstable training if you just use all the errors accumulated in 60,000 images to update the model rather than gradually update the model. In layman terms, imagine you accumulated errors for a student taking an exam with 60,000 questions and punish the student all at the same time. It is much harder for the student to learn compared to letting the student learn it made mistakes and did well in smaller batches of questions like mini-tests! If we have 60,000 images and we want a batch size of 100, then we would have 600 iterations where each iteration involves passing 600 images to the model and getting their respective predictions. 60000 / 100 600.0 Epochs An epoch means that you have successfully passed the whole training set, 60,000 images, to the model. Continuing our example above, an epoch consists of 600 iterations. If we want to go through the whole dataset 5 times (5 epochs) for the model to learn, then we need 3000 iterations (600 x 5). 600 * 5 3000.0 Bringing batch size, iterations and epochs together As we have gone through above, we want to have 5 epochs, where each epoch would have 600 iterations and each iteration has a batch size of 100. Because we want 5 epochs, we need a total of 3000 iterations. batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) Step 3: Create Model Class \u00b6 Creating our feedforward neural network Compared to logistic regression with only a single linear layer, we know for an FNN we need an additional linear layer and non-linear layer. This translates to just 4 more lines of code! class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . sigmoid = nn . Sigmoid () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function # LINEAR out = self . fc1 ( x ) # Non-linearity # NON-LINEAR out = self . sigmoid ( out ) # Linear function (readout) # LINEAR out = self . fc2 ( out ) return out Step 4: Instantiate Model Class \u00b6 Input dimension: 784 Size of image \\(28 \\times 28 = 784\\) Output dimension: 10 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 Hidden dimension: 100 Can be any number Similar term Number of neurons Number of non-linear activation functions Instantiating our model class Our input size is determined by the size of the image (numbers ranging from 0 to 9) which has a width of 28 pixels and a height of 28 pixels. Hence the size of our input is 784 (28 x 28). Our output size is what we are trying to predict. When we pass an image to our model, it will try to predict if it's 0, 1, 2, 3, 4, 5, 6, 7, 8, or 9. That is a total of 10 classes, hence we have an output size of 10. Now the tricky part is in determining our hidden layer size, that is the size of our first linear layer prior to the non-linear layer. This can be any number, a larger number implies a bigger model with more parameters. Intuitively we think a bigger model equates to a better model, but a bigger model requires more training samples to learn and converge to a good model (also called curse of dimensionality). Hence, it is wise to pick the model size for the problem at hand. Because it is a simple problem of recognizing digits, we typically would not need a big model to achieve state-of-the-art results. On the flipside, too small of a hidden size would mean there would be insufficient model capacity to predict competently. In layman terms, too small of a capacity implies a smaller brain capacity so no matter how many training samples you give it, it has a maximum capacity in terms of its predictive power. input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) Step 5: Instantiate Loss Class \u00b6 Feedforward Neural Network: Cross Entropy Loss Logistic Regression : Cross Entropy Loss Linear Regression : MSE Loss class This is exactly the same as what we did in logistic regression. Because we are going through a classification problem, cross entropy function is required to compute the loss between our softmax outputs and our binary labels. criterion = nn . CrossEntropyLoss () Step 6: Instantiate Optimizer Class \u00b6 Simplified equation \\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\) \\(\\theta\\) : parameters (our tensors with gradient accumulation capabilities) \\(\\eta\\) : learning rate (how fast we want to learn) \\(\\nabla_\\theta\\) : parameters' gradients Even simplier equation parameters = parameters - learning_rate * parameters_gradients At every iteration, we update our model's parameters Optimizer class Learning rate determines how fast the algorithm learns. Too small and the algorithm learns too slowly, too large and the algorithm learns too fast resulting in instabilities. Intuitively, we would think a larger learning rate would be better because we learn faster. But that's not true. Imagine we pass 10 images to a human to learn how to recognize whether the image is a hot dog or not, and it got half right and half wrong. A well defined learning rate (neither too small or large) is equivalent to rewarding the human with a sweet for getting the first half right, and punishing the other half the human got wrong with a smack on the palm. A large learning rate would be equivalent to feeding a thousand sweets to the human and smacking a thousand times on the human's palm. This would lead in a very unstable learning environment. Similarly, we will observe that the algorithm's convergence path will be extremely unstable if you use a large learning rate without reducing it subsequently. We are using an optimization algorithm called Stochastic Gradient Descent (SGD) which is essentially what we covered above on calculating the parameters' gradients multiplied by the learning rate then using it to update our parameters gradually. There's an in-depth analysis of various optimization algorithms on top of SGD in another section. learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) Parameters In-Depth \u00b6 Linear layers' parameters In a simple linear layer it's \\(Y = AX + B\\) , and our parameters are \\(A\\) and bias \\(B\\) . Hence, each linear layer would have 2 groups of parameters \\(A\\) and \\(B\\) . It is critical to take note that our non-linear layers have no parameters to update. They are merely mathematical functions performed on \\(Y\\) , the output of our linear layers. This would return a Python generator object, so you need to call list on the generator object to access anything meaningful. print ( model . parameters ()) Here we call list on the generator object and getting the length of the list. This would return 4 because we've 2 linear layers, and each layer has 2 groups of parameters \\(A\\) and \\(b\\) . print ( len ( list ( model . parameters ()))) Our first linear layer parameters, \\(A_1\\) , would be of size 100 x 784. This is because we've an input size of 784 (28 x 28) and a hidden size of 100. # FC 1 Parameters print ( list ( model . parameters ())[ 0 ] . size ()) Our first linear layer bias parameters, \\(B_1\\) , would be of size 100 which is our hidden size. # FC 1 Bias Parameters print ( list ( model . parameters ())[ 1 ] . size ()) Our second linear layer is our readout layer, where the parameters \\(A_2\\) would be of size 10 x 100. This is because our output size is 10 and hidden size is 100. # FC 2 Parameters print ( list ( model . parameters ())[ 2 ] . size ()) Likewise our readout layer's bias \\(B_1\\) would just be 10, the size of our output. # FC 2 Bias Parameters print ( list ( model . parameters ())[ 3 ] . size ()) The diagram below shows the interaction amongst our input \\(X\\) and our linear layers' parameters \\(A_1\\) , \\(B_1\\) , \\(A_2\\) , and \\(B_2\\) to reach to the final size of 10 x 1. If you're still unfamiliar with matrix product, go ahead and review the previous quick lesson where we covered it in logistic regression . < generator object Module . parameters at 0x7f1d530fa678 > 4 torch . Size ([ 100 , 784 ]) torch . Size ([ 100 ]) torch . Size ([ 10 , 100 ]) torch . Size ([ 10 ]) Step 7: Train Model \u00b6 Process Convert inputs to tensors with gradient accumulation capabilities Clear gradient buffers Get output given inputs Get loss Get gradients w.r.t. parameters Update parameters using gradients parameters = parameters - learning_rate * parameters_gradients REPEAT 7-step training process iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 0.6457265615463257 . Accuracy : 85 Iteration : 1000. Loss : 0.39627206325531006 . Accuracy : 89 Iteration : 1500. Loss : 0.2831554412841797 . Accuracy : 90 Iteration : 2000. Loss : 0.4409525394439697 . Accuracy : 91 Iteration : 2500. Loss : 0.2397005707025528 . Accuracy : 91 Iteration : 3000. Loss : 0.3160165846347809 . Accuracy : 91 Model B: 1 Hidden Layer Feedforward Neural Network (Tanh Activation) \u00b6 Steps \u00b6 Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 1-layer FNN with Tanh Activation The only difference here compared to previously is that we are using Tanh activation instead of Sigmoid activation. This affects step 3. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . tanh = nn . Tanh () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . tanh ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 0.4128190577030182 . Accuracy : 91 Iteration : 1000. Loss : 0.14497484266757965 . Accuracy : 92 Iteration : 1500. Loss : 0.272532194852829 . Accuracy : 93 Iteration : 2000. Loss : 0.2758277952671051 . Accuracy : 94 Iteration : 2500. Loss : 0.1603182554244995 . Accuracy : 94 Iteration : 3000. Loss : 0.08848697692155838 . Accuracy : 95 Model C: 1 Hidden Layer Feedforward Neural Network (ReLU Activation) \u00b6 Steps \u00b6 Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 1-layer FNN with ReLU Activation The only difference again is in using ReLU activation and it affects step 3. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 0.3179700970649719 . Accuracy : 91 Iteration : 1000. Loss : 0.17288273572921753 . Accuracy : 93 Iteration : 1500. Loss : 0.16829034686088562 . Accuracy : 94 Iteration : 2000. Loss : 0.25494423508644104 . Accuracy : 94 Iteration : 2500. Loss : 0.16818439960479736 . Accuracy : 95 Iteration : 3000. Loss : 0.11110792309045792 . Accuracy : 95 Model D: 2 Hidden Layer Feedforward Neural Network (ReLU Activation) \u00b6 Steps \u00b6 Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 2-layer FNN with ReLU Activation This is a bigger difference that increases your model's capacity by adding another linear layer and non-linear layer which affects step 3. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function 1: 784 --> 100 self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity 1 self . relu1 = nn . ReLU () # Linear function 2: 100 --> 100 self . fc2 = nn . Linear ( hidden_dim , hidden_dim ) # Non-linearity 2 self . relu2 = nn . ReLU () # Linear function 3 (readout): 100 --> 10 self . fc3 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function 1 out = self . fc1 ( x ) # Non-linearity 1 out = self . relu1 ( out ) # Linear function 2 out = self . fc2 ( out ) # Non-linearity 2 out = self . relu2 ( out ) # Linear function 3 (readout) out = self . fc3 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () labels = labels # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 0.2995373010635376 . Accuracy : 91 Iteration : 1000. Loss : 0.3924565613269806 . Accuracy : 93 Iteration : 1500. Loss : 0.1283276081085205 . Accuracy : 94 Iteration : 2000. Loss : 0.10905527323484421 . Accuracy : 95 Iteration : 2500. Loss : 0.11943754553794861 . Accuracy : 96 Iteration : 3000. Loss : 0.15632082521915436 . Accuracy : 96 Model E: 3 Hidden Layer Feedforward Neural Network (ReLU Activation) \u00b6 Steps \u00b6 Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 3-layer FNN with ReLU Activation Let's add one more layer! Bigger model capacity. But will it be better? Remember what we talked about on curse of dimensionality? import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function 1: 784 --> 100 self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity 1 self . relu1 = nn . ReLU () # Linear function 2: 100 --> 100 self . fc2 = nn . Linear ( hidden_dim , hidden_dim ) # Non-linearity 2 self . relu2 = nn . ReLU () # Linear function 3: 100 --> 100 self . fc3 = nn . Linear ( hidden_dim , hidden_dim ) # Non-linearity 3 self . relu3 = nn . ReLU () # Linear function 4 (readout): 100 --> 10 self . fc4 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function 1 out = self . fc1 ( x ) # Non-linearity 1 out = self . relu1 ( out ) # Linear function 2 out = self . fc2 ( out ) # Non-linearity 2 out = self . relu2 ( out ) # Linear function 2 out = self . fc3 ( out ) # Non-linearity 2 out = self . relu3 ( out ) # Linear function 4 (readout) out = self . fc4 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 0.33234935998916626 . Accuracy : 89 Iteration : 1000. Loss : 0.3098006248474121 . Accuracy : 94 Iteration : 1500. Loss : 0.12461677193641663 . Accuracy : 95 Iteration : 2000. Loss : 0.14346086978912354 . Accuracy : 96 Iteration : 2500. Loss : 0.03763459622859955 . Accuracy : 96 Iteration : 3000. Loss : 0.1397182047367096 . Accuracy : 97 General Comments on FNNs \u00b6 2 ways to expand a neural network More non-linear activation units (neurons) More hidden layers Cons Need a larger dataset Curse of dimensionality Does not necessarily mean higher accuracy 3. Building a Feedforward Neural Network with PyTorch (GPU) \u00b6 GPU: 2 things must be on GPU - model - tensors Steps \u00b6 Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 3-layer FNN with ReLU Activation on GPU Only step 4 and 7 of the CPU code will be affected and it's a simple change. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function 1: 784 --> 100 self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity 1 self . relu1 = nn . ReLU () # Linear function 2: 100 --> 100 self . fc2 = nn . Linear ( hidden_dim , hidden_dim ) # Non-linearity 2 self . relu2 = nn . ReLU () # Linear function 3: 100 --> 100 self . fc3 = nn . Linear ( hidden_dim , hidden_dim ) # Non-linearity 3 self . relu3 = nn . ReLU () # Linear function 4 (readout): 100 --> 10 self . fc4 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function 1 out = self . fc1 ( x ) # Non-linearity 1 out = self . relu1 ( out ) # Linear function 2 out = self . fc2 ( out ) # Non-linearity 2 out = self . relu2 ( out ) # Linear function 2 out = self . fc3 ( out ) # Non-linearity 2 out = self . relu3 ( out ) # Linear function 4 (readout) out = self . fc4 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ####################### # USE GPU FOR MODEL # ####################### device = torch . device ( \"cuda:0\" if torch . cuda . is_available () else \"cpu\" ) model . to ( device ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): ####################### # USE GPU FOR MODEL # ####################### images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () . to ( device ) labels = labels . to ( device ) # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : ####################### # USE GPU FOR MODEL # ####################### images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () . to ( device ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) ####################### # USE GPU FOR MODEL # ####################### # Total correct predictions if torch . cuda . is_available (): correct += ( predicted . cpu () == labels . cpu ()) . sum () else : correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 0.3877025246620178 . Accuracy : 90 Iteration : 1000. Loss : 0.1337055265903473 . Accuracy : 93 Iteration : 1500. Loss : 0.2038637101650238 . Accuracy : 95 Iteration : 2000. Loss : 0.17892278730869293 . Accuracy : 95 Iteration : 2500. Loss : 0.14455552399158478 . Accuracy : 96 Iteration : 3000. Loss : 0.024540524929761887 . Accuracy : 96 Alternative Term of Neural Network The alternative term is Universal Function Approximator . This is because ultimately we are trying to find a function that maps our input, \\(X\\) , to our output, \\(y\\) . Summary \u00b6 We've learnt to... Success Logistic Regression Problems for Non-Linear Functions Representation Cannot represent non-linear functions $ y = 4x_1 + 2x_2^2 +3x_3^3 $ $ y = x_1x_2$ Introduced Non-Linearity to Logistic Regression to form a Neural Network Types of Non-Linearity Sigmoid Tanh ReLU Feedforward Neural Network Models Model A: 1 hidden layer ( sigmoid activation) Model B: 1 hidden layer ( tanh activation) Model C: 1 hidden layer ( ReLU activation) Model D: 2 hidden layers (ReLU activation) Model E: 3 hidden layers (ReLU activation) Models Variation in Code Modifying only step 3 Ways to Expand Model\u2019s Capacity More non-linear activation units ( neurons ) More hidden layers Cons of Expanding Capacity Need more data Does not necessarily mean higher accuracy GPU Code 2 things on GPU model tensors with gradient accumulation capabilities Modifying only Step 4 & Step 7 7 Step Model Building Recap Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model Citation \u00b6 If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Feedforward Neural Networks (FNN)"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#feedforward-neural-network-with-pytorch","text":"Run Jupyter Notebook You can run the code for this section in this jupyter notebook link .","title":"Feedforward Neural Network with PyTorch"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#about-feedforward-neural-network","text":"","title":"About Feedforward Neural Network"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#logistic-regression-transition-to-neural-networks","text":"","title":"Logistic Regression Transition to Neural Networks"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#logistic-regression-review","text":"Define logistic regression model Import our relevant torch modules. import torch import torch.nn as nn Define our model class. class LogisticRegressionModel ( nn . Module ): def __init__ ( self , input_dim , output_dim ): super ( LogisticRegressionModel , self ) . __init__ () self . linear = nn . Linear ( input_dim , output_dim ) def forward ( self , x ): out = self . linear ( x ) return out Instantiate the logistic regression model. input_dim = 28 * 28 output_dim = 10 model = LogisticRegressionModel ( input_dim , output_dim ) When we inspect the model, we would have an input size of 784 (derived from 28 x 28) and output size of 10 (which is the number of classes we are classifying from 0 to 9). print ( model ) LogisticRegressionModel ( ( linear ): Linear ( in_features = 784 , out_features = 10 , bias = True ) )","title":"Logistic Regression Review"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#logistic-regression-problems","text":"Can represent linear functions well \\(y = 2x + 3\\) \\(y = x_1 + x_2\\) \\(y = x_1 + 3x_2 + 4x_3\\) Cannot represent non-linear functions \\(y = 4x_1 + 2x_2^2 +3x_3^3\\) \\(y = x_1x_2\\)","title":"Logistic Regression Problems"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#introducing-a-non-linear-function","text":"","title":"Introducing a Non-linear Function"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#non-linear-function-in-depth","text":"Function: takes a number & perform mathematical operation Common Types of Non-linearity ReLUs (Rectified Linear Units) Sigmoid Tanh","title":"Non-linear Function In-Depth"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#sigmoid-logistic","text":"\\(\\sigma(x) = \\frac{1}{1 + e^{-x}}\\) Input number \\(\\rightarrow\\) [0, 1] Large negative number \\(\\rightarrow\\) 0 Large positive number \\(\\rightarrow\\) 1 Cons: Activation saturates at 0 or 1 with gradients \\(\\approx\\) 0 No signal to update weights \\(\\rightarrow\\) cannot learn Solution: Have to carefully initialize weights to prevent this Outputs not centered around 0 If output always positive \\(\\rightarrow\\) gradients always positive or negative \\(\\rightarrow\\) bad for gradient updates","title":"Sigmoid (Logistic)"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#tanh","text":"\\(\\tanh(x) = 2 \\sigma(2x) -1\\) A scaled sigmoid function Input number \\(\\rightarrow\\) [-1, 1] Cons: Activation saturates at 0 or 1 with gradients \\(\\approx\\) 0 No signal to update weights \\(\\rightarrow\\) cannot learn Solution : Have to carefully initialize weights to prevent this","title":"Tanh"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#relus","text":"\\(f(x) = \\max(0, x)\\) Pros: Accelerates convergence \\(\\rightarrow\\) train faster Less computationally expensive operation compared to Sigmoid/Tanh exponentials Cons: Many ReLU units \"die\" \\(\\rightarrow\\) gradients = 0 forever Solution : careful learning rate choice","title":"ReLUs"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#building-a-feedforward-neural-network-with-pytorch","text":"","title":"Building a Feedforward Neural Network with PyTorch"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#model-a-1-hidden-layer-feedforward-neural-network-sigmoid-activation","text":"","title":"Model A: 1 Hidden Layer Feedforward Neural Network (Sigmoid Activation)"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#steps","text":"Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model","title":"Steps"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#step-1-loading-mnist-train-dataset","text":"Images from 1 to 9 Similar to what we did in logistic regression, we will be using the same MNIST dataset where we load our training and testing datasets. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ())","title":"Step 1: Loading MNIST Train Dataset"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#step-2-make-dataset-iterable","text":"Batch sizes and iterations Because we have 60000 training samples (images), we need to split them up to small groups (batches) and pass these batches of samples to our feedforward neural network subsesquently. There are a few reasons why we split them into batches. Passing your whole dataset as a single batch would: (1) require a lot of RAM/VRAM on your CPU/GPU and this might result in Out-of-Memory (OOM) errors. (2) cause unstable training if you just use all the errors accumulated in 60,000 images to update the model rather than gradually update the model. In layman terms, imagine you accumulated errors for a student taking an exam with 60,000 questions and punish the student all at the same time. It is much harder for the student to learn compared to letting the student learn it made mistakes and did well in smaller batches of questions like mini-tests! If we have 60,000 images and we want a batch size of 100, then we would have 600 iterations where each iteration involves passing 600 images to the model and getting their respective predictions. 60000 / 100 600.0 Epochs An epoch means that you have successfully passed the whole training set, 60,000 images, to the model. Continuing our example above, an epoch consists of 600 iterations. If we want to go through the whole dataset 5 times (5 epochs) for the model to learn, then we need 3000 iterations (600 x 5). 600 * 5 3000.0 Bringing batch size, iterations and epochs together As we have gone through above, we want to have 5 epochs, where each epoch would have 600 iterations and each iteration has a batch size of 100. Because we want 5 epochs, we need a total of 3000 iterations. batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False )","title":"Step 2: Make Dataset Iterable"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#step-3-create-model-class","text":"Creating our feedforward neural network Compared to logistic regression with only a single linear layer, we know for an FNN we need an additional linear layer and non-linear layer. This translates to just 4 more lines of code! class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . sigmoid = nn . Sigmoid () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function # LINEAR out = self . fc1 ( x ) # Non-linearity # NON-LINEAR out = self . sigmoid ( out ) # Linear function (readout) # LINEAR out = self . fc2 ( out ) return out","title":"Step 3: Create Model Class"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#step-4-instantiate-model-class","text":"Input dimension: 784 Size of image \\(28 \\times 28 = 784\\) Output dimension: 10 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 Hidden dimension: 100 Can be any number Similar term Number of neurons Number of non-linear activation functions Instantiating our model class Our input size is determined by the size of the image (numbers ranging from 0 to 9) which has a width of 28 pixels and a height of 28 pixels. Hence the size of our input is 784 (28 x 28). Our output size is what we are trying to predict. When we pass an image to our model, it will try to predict if it's 0, 1, 2, 3, 4, 5, 6, 7, 8, or 9. That is a total of 10 classes, hence we have an output size of 10. Now the tricky part is in determining our hidden layer size, that is the size of our first linear layer prior to the non-linear layer. This can be any number, a larger number implies a bigger model with more parameters. Intuitively we think a bigger model equates to a better model, but a bigger model requires more training samples to learn and converge to a good model (also called curse of dimensionality). Hence, it is wise to pick the model size for the problem at hand. Because it is a simple problem of recognizing digits, we typically would not need a big model to achieve state-of-the-art results. On the flipside, too small of a hidden size would mean there would be insufficient model capacity to predict competently. In layman terms, too small of a capacity implies a smaller brain capacity so no matter how many training samples you give it, it has a maximum capacity in terms of its predictive power. input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim )","title":"Step 4: Instantiate Model Class"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#step-5-instantiate-loss-class","text":"Feedforward Neural Network: Cross Entropy Loss Logistic Regression : Cross Entropy Loss Linear Regression : MSE Loss class This is exactly the same as what we did in logistic regression. Because we are going through a classification problem, cross entropy function is required to compute the loss between our softmax outputs and our binary labels. criterion = nn . CrossEntropyLoss ()","title":"Step 5: Instantiate Loss Class"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#step-6-instantiate-optimizer-class","text":"Simplified equation \\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\) \\(\\theta\\) : parameters (our tensors with gradient accumulation capabilities) \\(\\eta\\) : learning rate (how fast we want to learn) \\(\\nabla_\\theta\\) : parameters' gradients Even simplier equation parameters = parameters - learning_rate * parameters_gradients At every iteration, we update our model's parameters Optimizer class Learning rate determines how fast the algorithm learns. Too small and the algorithm learns too slowly, too large and the algorithm learns too fast resulting in instabilities. Intuitively, we would think a larger learning rate would be better because we learn faster. But that's not true. Imagine we pass 10 images to a human to learn how to recognize whether the image is a hot dog or not, and it got half right and half wrong. A well defined learning rate (neither too small or large) is equivalent to rewarding the human with a sweet for getting the first half right, and punishing the other half the human got wrong with a smack on the palm. A large learning rate would be equivalent to feeding a thousand sweets to the human and smacking a thousand times on the human's palm. This would lead in a very unstable learning environment. Similarly, we will observe that the algorithm's convergence path will be extremely unstable if you use a large learning rate without reducing it subsequently. We are using an optimization algorithm called Stochastic Gradient Descent (SGD) which is essentially what we covered above on calculating the parameters' gradients multiplied by the learning rate then using it to update our parameters gradually. There's an in-depth analysis of various optimization algorithms on top of SGD in another section. learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate )","title":"Step 6: Instantiate Optimizer Class"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#parameters-in-depth","text":"Linear layers' parameters In a simple linear layer it's \\(Y = AX + B\\) , and our parameters are \\(A\\) and bias \\(B\\) . Hence, each linear layer would have 2 groups of parameters \\(A\\) and \\(B\\) . It is critical to take note that our non-linear layers have no parameters to update. They are merely mathematical functions performed on \\(Y\\) , the output of our linear layers. This would return a Python generator object, so you need to call list on the generator object to access anything meaningful. print ( model . parameters ()) Here we call list on the generator object and getting the length of the list. This would return 4 because we've 2 linear layers, and each layer has 2 groups of parameters \\(A\\) and \\(b\\) . print ( len ( list ( model . parameters ()))) Our first linear layer parameters, \\(A_1\\) , would be of size 100 x 784. This is because we've an input size of 784 (28 x 28) and a hidden size of 100. # FC 1 Parameters print ( list ( model . parameters ())[ 0 ] . size ()) Our first linear layer bias parameters, \\(B_1\\) , would be of size 100 which is our hidden size. # FC 1 Bias Parameters print ( list ( model . parameters ())[ 1 ] . size ()) Our second linear layer is our readout layer, where the parameters \\(A_2\\) would be of size 10 x 100. This is because our output size is 10 and hidden size is 100. # FC 2 Parameters print ( list ( model . parameters ())[ 2 ] . size ()) Likewise our readout layer's bias \\(B_1\\) would just be 10, the size of our output. # FC 2 Bias Parameters print ( list ( model . parameters ())[ 3 ] . size ()) The diagram below shows the interaction amongst our input \\(X\\) and our linear layers' parameters \\(A_1\\) , \\(B_1\\) , \\(A_2\\) , and \\(B_2\\) to reach to the final size of 10 x 1. If you're still unfamiliar with matrix product, go ahead and review the previous quick lesson where we covered it in logistic regression . < generator object Module . parameters at 0x7f1d530fa678 > 4 torch . Size ([ 100 , 784 ]) torch . Size ([ 100 ]) torch . Size ([ 10 , 100 ]) torch . Size ([ 10 ])","title":"Parameters In-Depth"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#step-7-train-model","text":"Process Convert inputs to tensors with gradient accumulation capabilities Clear gradient buffers Get output given inputs Get loss Get gradients w.r.t. parameters Update parameters using gradients parameters = parameters - learning_rate * parameters_gradients REPEAT 7-step training process iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 0.6457265615463257 . Accuracy : 85 Iteration : 1000. Loss : 0.39627206325531006 . Accuracy : 89 Iteration : 1500. Loss : 0.2831554412841797 . Accuracy : 90 Iteration : 2000. Loss : 0.4409525394439697 . Accuracy : 91 Iteration : 2500. Loss : 0.2397005707025528 . Accuracy : 91 Iteration : 3000. Loss : 0.3160165846347809 . Accuracy : 91","title":"Step 7: Train Model"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#model-b-1-hidden-layer-feedforward-neural-network-tanh-activation","text":"","title":"Model B: 1 Hidden Layer Feedforward Neural Network (Tanh Activation)"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#steps_1","text":"Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 1-layer FNN with Tanh Activation The only difference here compared to previously is that we are using Tanh activation instead of Sigmoid activation. This affects step 3. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . tanh = nn . Tanh () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . tanh ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 0.4128190577030182 . Accuracy : 91 Iteration : 1000. Loss : 0.14497484266757965 . Accuracy : 92 Iteration : 1500. Loss : 0.272532194852829 . Accuracy : 93 Iteration : 2000. Loss : 0.2758277952671051 . Accuracy : 94 Iteration : 2500. Loss : 0.1603182554244995 . Accuracy : 94 Iteration : 3000. Loss : 0.08848697692155838 . Accuracy : 95","title":"Steps"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#model-c-1-hidden-layer-feedforward-neural-network-relu-activation","text":"","title":"Model C: 1 Hidden Layer Feedforward Neural Network (ReLU Activation)"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#steps_2","text":"Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 1-layer FNN with ReLU Activation The only difference again is in using ReLU activation and it affects step 3. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity self . relu = nn . ReLU () # Linear function (readout) self . fc2 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function out = self . fc1 ( x ) # Non-linearity out = self . relu ( out ) # Linear function (readout) out = self . fc2 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 0.3179700970649719 . Accuracy : 91 Iteration : 1000. Loss : 0.17288273572921753 . Accuracy : 93 Iteration : 1500. Loss : 0.16829034686088562 . Accuracy : 94 Iteration : 2000. Loss : 0.25494423508644104 . Accuracy : 94 Iteration : 2500. Loss : 0.16818439960479736 . Accuracy : 95 Iteration : 3000. Loss : 0.11110792309045792 . Accuracy : 95","title":"Steps"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#model-d-2-hidden-layer-feedforward-neural-network-relu-activation","text":"","title":"Model D: 2 Hidden Layer Feedforward Neural Network (ReLU Activation)"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#steps_3","text":"Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 2-layer FNN with ReLU Activation This is a bigger difference that increases your model's capacity by adding another linear layer and non-linear layer which affects step 3. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function 1: 784 --> 100 self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity 1 self . relu1 = nn . ReLU () # Linear function 2: 100 --> 100 self . fc2 = nn . Linear ( hidden_dim , hidden_dim ) # Non-linearity 2 self . relu2 = nn . ReLU () # Linear function 3 (readout): 100 --> 10 self . fc3 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function 1 out = self . fc1 ( x ) # Non-linearity 1 out = self . relu1 ( out ) # Linear function 2 out = self . fc2 ( out ) # Non-linearity 2 out = self . relu2 ( out ) # Linear function 3 (readout) out = self . fc3 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () labels = labels # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 0.2995373010635376 . Accuracy : 91 Iteration : 1000. Loss : 0.3924565613269806 . Accuracy : 93 Iteration : 1500. Loss : 0.1283276081085205 . Accuracy : 94 Iteration : 2000. Loss : 0.10905527323484421 . Accuracy : 95 Iteration : 2500. Loss : 0.11943754553794861 . Accuracy : 96 Iteration : 3000. Loss : 0.15632082521915436 . Accuracy : 96","title":"Steps"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#model-e-3-hidden-layer-feedforward-neural-network-relu-activation","text":"","title":"Model E: 3 Hidden Layer Feedforward Neural Network (ReLU Activation)"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#steps_4","text":"Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 3-layer FNN with ReLU Activation Let's add one more layer! Bigger model capacity. But will it be better? Remember what we talked about on curse of dimensionality? import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function 1: 784 --> 100 self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity 1 self . relu1 = nn . ReLU () # Linear function 2: 100 --> 100 self . fc2 = nn . Linear ( hidden_dim , hidden_dim ) # Non-linearity 2 self . relu2 = nn . ReLU () # Linear function 3: 100 --> 100 self . fc3 = nn . Linear ( hidden_dim , hidden_dim ) # Non-linearity 3 self . relu3 = nn . ReLU () # Linear function 4 (readout): 100 --> 10 self . fc4 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function 1 out = self . fc1 ( x ) # Non-linearity 1 out = self . relu1 ( out ) # Linear function 2 out = self . fc2 ( out ) # Non-linearity 2 out = self . relu2 ( out ) # Linear function 2 out = self . fc3 ( out ) # Non-linearity 2 out = self . relu3 ( out ) # Linear function 4 (readout) out = self . fc4 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images with gradient accumulation capabilities images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 0.33234935998916626 . Accuracy : 89 Iteration : 1000. Loss : 0.3098006248474121 . Accuracy : 94 Iteration : 1500. Loss : 0.12461677193641663 . Accuracy : 95 Iteration : 2000. Loss : 0.14346086978912354 . Accuracy : 96 Iteration : 2500. Loss : 0.03763459622859955 . Accuracy : 96 Iteration : 3000. Loss : 0.1397182047367096 . Accuracy : 97","title":"Steps"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#general-comments-on-fnns","text":"2 ways to expand a neural network More non-linear activation units (neurons) More hidden layers Cons Need a larger dataset Curse of dimensionality Does not necessarily mean higher accuracy","title":"General Comments on FNNs"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#3-building-a-feedforward-neural-network-with-pytorch-gpu","text":"GPU: 2 things must be on GPU - model - tensors","title":"3. Building a Feedforward Neural Network with PyTorch (GPU)"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#steps_5","text":"Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 3-layer FNN with ReLU Activation on GPU Only step 4 and 7 of the CPU code will be affected and it's a simple change. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class FeedforwardNeuralNetModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , output_dim ): super ( FeedforwardNeuralNetModel , self ) . __init__ () # Linear function 1: 784 --> 100 self . fc1 = nn . Linear ( input_dim , hidden_dim ) # Non-linearity 1 self . relu1 = nn . ReLU () # Linear function 2: 100 --> 100 self . fc2 = nn . Linear ( hidden_dim , hidden_dim ) # Non-linearity 2 self . relu2 = nn . ReLU () # Linear function 3: 100 --> 100 self . fc3 = nn . Linear ( hidden_dim , hidden_dim ) # Non-linearity 3 self . relu3 = nn . ReLU () # Linear function 4 (readout): 100 --> 10 self . fc4 = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Linear function 1 out = self . fc1 ( x ) # Non-linearity 1 out = self . relu1 ( out ) # Linear function 2 out = self . fc2 ( out ) # Non-linearity 2 out = self . relu2 ( out ) # Linear function 2 out = self . fc3 ( out ) # Non-linearity 2 out = self . relu3 ( out ) # Linear function 4 (readout) out = self . fc4 ( out ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 hidden_dim = 100 output_dim = 10 model = FeedforwardNeuralNetModel ( input_dim , hidden_dim , output_dim ) ####################### # USE GPU FOR MODEL # ####################### device = torch . device ( \"cuda:0\" if torch . cuda . is_available () else \"cpu\" ) model . to ( device ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): ####################### # USE GPU FOR MODEL # ####################### images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () . to ( device ) labels = labels . to ( device ) # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : ####################### # USE GPU FOR MODEL # ####################### images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () . to ( device ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) ####################### # USE GPU FOR MODEL # ####################### # Total correct predictions if torch . cuda . is_available (): correct += ( predicted . cpu () == labels . cpu ()) . sum () else : correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 0.3877025246620178 . Accuracy : 90 Iteration : 1000. Loss : 0.1337055265903473 . Accuracy : 93 Iteration : 1500. Loss : 0.2038637101650238 . Accuracy : 95 Iteration : 2000. Loss : 0.17892278730869293 . Accuracy : 95 Iteration : 2500. Loss : 0.14455552399158478 . Accuracy : 96 Iteration : 3000. Loss : 0.024540524929761887 . Accuracy : 96 Alternative Term of Neural Network The alternative term is Universal Function Approximator . This is because ultimately we are trying to find a function that maps our input, \\(X\\) , to our output, \\(y\\) .","title":"Steps"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#summary","text":"We've learnt to... Success Logistic Regression Problems for Non-Linear Functions Representation Cannot represent non-linear functions $ y = 4x_1 + 2x_2^2 +3x_3^3 $ $ y = x_1x_2$ Introduced Non-Linearity to Logistic Regression to form a Neural Network Types of Non-Linearity Sigmoid Tanh ReLU Feedforward Neural Network Models Model A: 1 hidden layer ( sigmoid activation) Model B: 1 hidden layer ( tanh activation) Model C: 1 hidden layer ( ReLU activation) Model D: 2 hidden layers (ReLU activation) Model E: 3 hidden layers (ReLU activation) Models Variation in Code Modifying only step 3 Ways to Expand Model\u2019s Capacity More non-linear activation units ( neurons ) More hidden layers Cons of Expanding Capacity Need more data Does not necessarily mean higher accuracy GPU Code 2 things on GPU model tensors with gradient accumulation capabilities Modifying only Step 4 & Step 7 7 Step Model Building Recap Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model","title":"Summary"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#citation","text":"If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Citation"},{"location":"deep_learning/practical_pytorch/pytorch_gradients/","text":"Gradients with PyTorch \u00b6 Run Jupyter Notebook You can run the code for this section in this jupyter notebook link . Tensors with Gradients \u00b6 Creating Tensors with Gradients \u00b6 Allows accumulation of gradients Method 1: Create tensor with gradients It is very similar to creating a tensor, all you need to do is to add an additional argument. import torch a = torch . ones (( 2 , 2 ), requires_grad = True ) a tensor ([[ 1. , 1. ], [ 1. , 1. ]]) Check if tensor requires gradients This should return True otherwise you've not done it right. a . requires_grad True Method 2: Create tensor with gradients This allows you to create a tensor as usual then an additional line to allow it to accumulate gradients. # Normal way of creating gradients a = torch . ones (( 2 , 2 )) # Requires gradient a . requires_grad_ () # Check if requires gradient a . requires_grad True A tensor without gradients just for comparison If you do not do either of the methods above, you'll realize you will get False for checking for gradients. # Not a variable no_gradient = torch . ones ( 2 , 2 ) no_gradient . requires_grad False Tensor with gradients addition operation # Behaves similarly to tensors b = torch . ones (( 2 , 2 ), requires_grad = True ) print ( a + b ) print ( torch . add ( a , b )) tensor ([[ 2. , 2. ], [ 2. , 2. ]]) tensor ([[ 2. , 2. ], [ 2. , 2. ]]) Tensor with gradients multiplication operation As usual, the operations we learnt previously for tensors apply for tensors with gradients. Feel free to try divisions, mean or standard deviation! print ( a * b ) print ( torch . mul ( a , b )) tensor ([[ 1. , 1. ], [ 1. , 1. ]]) tensor ([[ 1. , 1. ], [ 1. , 1. ]]) Manually and Automatically Calculating Gradients \u00b6 What exactly is requires_grad ? - Allows calculation of gradients w.r.t. the tensor that all allows gradients accumulation \\[y_i = 5(x_i+1)^2\\] Create tensor of size 2x1 filled with 1's that requires gradient x = torch . ones ( 2 , requires_grad = True ) x tensor ([ 1. , 1. ]) Simple linear equation with x tensor created \\[y_i\\bigr\\rvert_{x_i=1} = 5(1 + 1)^2 = 5(2)^2 = 5(4) = 20\\] We should get a value of 20 by replicating this simple equation y = 5 * ( x + 1 ) ** 2 y tensor ([ 20. , 20. ]) Simple equation with y tensor Backward should be called only on a scalar (i.e. 1-element tensor) or with gradient w.r.t. the variable Let's reduce y to a scalar then... \\[o = \\frac{1}{2}\\sum_i y_i\\] As you can see above, we've a tensor filled with 20's, so average them would return 20 o = ( 1 / 2 ) * torch . sum ( y ) o tensor ( 20. ) Calculating first derivative Recap y equation : \\(y_i = 5(x_i+1)^2\\) Recap o equation : \\(o = \\frac{1}{2}\\sum_i y_i\\) Substitute y into o equation : \\(o = \\frac{1}{2} \\sum_i 5(x_i+1)^2\\) \\[\\frac{\\partial o}{\\partial x_i} = \\frac{1}{2}[10(x_i+1)]\\] \\[\\frac{\\partial o}{\\partial x_i}\\bigr\\rvert_{x_i=1} = \\frac{1}{2}[10(1 + 1)] = \\frac{10}{2}(2) = 10\\] We should expect to get 10, and it's so simple to do this with PyTorch with the following line... Get first derivative: o . backward () Print out first derivative: x . grad tensor ([ 10. , 10. ]) If x requires gradient and you create new objects with it, you get all gradients print ( x . requires_grad ) print ( y . requires_grad ) print ( o . requires_grad ) True True True Summary \u00b6 We've learnt to... Success Tensor with Gradients Wraps a tensor for gradient accumulation Gradients Define original equation Substitute equation with x values Reduce to scalar output, o through mean Calculate gradients with o.backward() Then access gradients of the x tensor with requires_grad through x.grad Citation \u00b6 If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Gradients"},{"location":"deep_learning/practical_pytorch/pytorch_gradients/#gradients-with-pytorch","text":"Run Jupyter Notebook You can run the code for this section in this jupyter notebook link .","title":"Gradients with PyTorch"},{"location":"deep_learning/practical_pytorch/pytorch_gradients/#tensors-with-gradients","text":"","title":"Tensors with Gradients"},{"location":"deep_learning/practical_pytorch/pytorch_gradients/#creating-tensors-with-gradients","text":"Allows accumulation of gradients Method 1: Create tensor with gradients It is very similar to creating a tensor, all you need to do is to add an additional argument. import torch a = torch . ones (( 2 , 2 ), requires_grad = True ) a tensor ([[ 1. , 1. ], [ 1. , 1. ]]) Check if tensor requires gradients This should return True otherwise you've not done it right. a . requires_grad True Method 2: Create tensor with gradients This allows you to create a tensor as usual then an additional line to allow it to accumulate gradients. # Normal way of creating gradients a = torch . ones (( 2 , 2 )) # Requires gradient a . requires_grad_ () # Check if requires gradient a . requires_grad True A tensor without gradients just for comparison If you do not do either of the methods above, you'll realize you will get False for checking for gradients. # Not a variable no_gradient = torch . ones ( 2 , 2 ) no_gradient . requires_grad False Tensor with gradients addition operation # Behaves similarly to tensors b = torch . ones (( 2 , 2 ), requires_grad = True ) print ( a + b ) print ( torch . add ( a , b )) tensor ([[ 2. , 2. ], [ 2. , 2. ]]) tensor ([[ 2. , 2. ], [ 2. , 2. ]]) Tensor with gradients multiplication operation As usual, the operations we learnt previously for tensors apply for tensors with gradients. Feel free to try divisions, mean or standard deviation! print ( a * b ) print ( torch . mul ( a , b )) tensor ([[ 1. , 1. ], [ 1. , 1. ]]) tensor ([[ 1. , 1. ], [ 1. , 1. ]])","title":"Creating Tensors with Gradients"},{"location":"deep_learning/practical_pytorch/pytorch_gradients/#manually-and-automatically-calculating-gradients","text":"What exactly is requires_grad ? - Allows calculation of gradients w.r.t. the tensor that all allows gradients accumulation \\[y_i = 5(x_i+1)^2\\] Create tensor of size 2x1 filled with 1's that requires gradient x = torch . ones ( 2 , requires_grad = True ) x tensor ([ 1. , 1. ]) Simple linear equation with x tensor created \\[y_i\\bigr\\rvert_{x_i=1} = 5(1 + 1)^2 = 5(2)^2 = 5(4) = 20\\] We should get a value of 20 by replicating this simple equation y = 5 * ( x + 1 ) ** 2 y tensor ([ 20. , 20. ]) Simple equation with y tensor Backward should be called only on a scalar (i.e. 1-element tensor) or with gradient w.r.t. the variable Let's reduce y to a scalar then... \\[o = \\frac{1}{2}\\sum_i y_i\\] As you can see above, we've a tensor filled with 20's, so average them would return 20 o = ( 1 / 2 ) * torch . sum ( y ) o tensor ( 20. ) Calculating first derivative Recap y equation : \\(y_i = 5(x_i+1)^2\\) Recap o equation : \\(o = \\frac{1}{2}\\sum_i y_i\\) Substitute y into o equation : \\(o = \\frac{1}{2} \\sum_i 5(x_i+1)^2\\) \\[\\frac{\\partial o}{\\partial x_i} = \\frac{1}{2}[10(x_i+1)]\\] \\[\\frac{\\partial o}{\\partial x_i}\\bigr\\rvert_{x_i=1} = \\frac{1}{2}[10(1 + 1)] = \\frac{10}{2}(2) = 10\\] We should expect to get 10, and it's so simple to do this with PyTorch with the following line... Get first derivative: o . backward () Print out first derivative: x . grad tensor ([ 10. , 10. ]) If x requires gradient and you create new objects with it, you get all gradients print ( x . requires_grad ) print ( y . requires_grad ) print ( o . requires_grad ) True True True","title":"Manually and Automatically Calculating Gradients"},{"location":"deep_learning/practical_pytorch/pytorch_gradients/#summary","text":"We've learnt to... Success Tensor with Gradients Wraps a tensor for gradient accumulation Gradients Define original equation Substitute equation with x values Reduce to scalar output, o through mean Calculate gradients with o.backward() Then access gradients of the x tensor with requires_grad through x.grad","title":"Summary"},{"location":"deep_learning/practical_pytorch/pytorch_gradients/#citation","text":"If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Citation"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/","text":"Linear Regression with PyTorch \u00b6 Run Jupyter Notebook You can run the code for this section in this jupyter notebook link . About Linear Regression \u00b6 Simple Linear Regression Basics \u00b6 Allows us to understand relationship between two continuous variables Example x: independent variable weight y: dependent variable height \\(y = \\alpha x + \\beta\\) Example of simple linear regression \u00b6 Create plot for simple linear regression Take note that this code is not important at all. It simply creates random data points and does a simple best-fit line to best approximate the underlying function if one even exists. import numpy as np import matplotlib.pyplot as plt % matplotlib inline # Creates 50 random x and y numbers np . random . seed ( 1 ) n = 50 x = np . random . randn ( n ) y = x * np . random . randn ( n ) # Makes the dots colorful colors = np . random . rand ( n ) # Plots best-fit line via polyfit plt . plot ( np . unique ( x ), np . poly1d ( np . polyfit ( x , y , 1 ))( np . unique ( x ))) # Plots the random x and y data points we created # Interestingly, alpha makes it more aesthetically pleasing plt . scatter ( x , y , c = colors , alpha = 0.5 ) plt . show () Aim of Linear Regression \u00b6 Minimize the distance between the points and the line ( \\(y = \\alpha x + \\beta\\) ) Adjusting Coefficient: \\(\\alpha\\) Bias/intercept: \\(\\beta\\) Building a Linear Regression Model with PyTorch \u00b6 Example \u00b6 Coefficient: \\(\\alpha = 2\\) Bias/intercept: \\(\\beta = 1\\) Equation: \\(y = 2x + 1\\) Building a Toy Dataset \u00b6 Create a list of values from 0 to 11 x_values = [ i for i in range ( 11 )] x_values [ 0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 ] Convert list of numbers to numpy array # Convert to numpy x_train = np . array ( x_values , dtype = np . float32 ) x_train . shape ( 11 ,) Convert to 2-dimensional array If you don't this you will get an error stating you need 2D. Simply just reshape accordingly if you ever face such errors down the road. # IMPORTANT: 2D required x_train = x_train . reshape ( - 1 , 1 ) x_train . shape ( 11 , 1 ) Create list of y values We want y values for every x value we have above. \\(y = 2x + 1\\) y_values = [ 2 * i + 1 for i in x_values ] y_values [ 1 , 3 , 5 , 7 , 9 , 11 , 13 , 15 , 17 , 19 , 21 ] Alternative to create list of y values If you're weak in list iterators, this might be an easier alternative. # In case you're weak in list iterators... y_values = [] for i in x_values : result = 2 * i + 1 y_values . append ( result ) y_values [ 1 , 3 , 5 , 7 , 9 , 11 , 13 , 15 , 17 , 19 , 21 ] Convert to numpy array You will slowly get a hang on how when you deal with PyTorch tensors, you just keep on making sure your raw data is in numpy form to make sure everything's good. y_train = np . array ( y_values , dtype = np . float32 ) y_train . shape ( 11 ,) Reshape y numpy array to 2-dimension # IMPORTANT: 2D required y_train = y_train . reshape ( - 1 , 1 ) y_train . shape ( 11 , 1 ) Building Model \u00b6 Critical Imports import torch import torch.nn as nn Create Model Linear model True Equation: \\(y = 2x + 1\\) Forward Example Input \\(x = 1\\) Output \\(\\hat y = ?\\) # Create class class LinearRegressionModel ( nn . Module ): def __init__ ( self , input_dim , output_dim ): super ( LinearRegressionModel , self ) . __init__ () self . linear = nn . Linear ( input_dim , output_dim ) def forward ( self , x ): out = self . linear ( x ) return out Instantiate Model Class input: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] desired output: [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21] input_dim = 1 output_dim = 1 model = LinearRegressionModel ( input_dim , output_dim ) Instantiate Loss Class MSE Loss: Mean Squared Error \\(MSE = \\frac{1}{n} \\sum_{i=1}^n(\\hat y_i - y_i)^2\\) \\(\\hat y\\) : prediction \\(y\\) : true value criterion = nn . MSELoss () Instantiate Optimizer Class Simplified equation \\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\) \\(\\theta\\) : parameters (our variables) \\(\\eta\\) : learning rate (how fast we want to learn) \\(\\nabla_\\theta\\) : parameters' gradients Even simplier equation parameters = parameters - learning_rate * parameters_gradients parameters: \\(\\alpha\\) and \\(\\beta\\) in \\(y = \\alpha x + \\beta\\) desired parameters: \\(\\alpha = 2\\) and \\(\\beta = 1\\) in \\(y = 2x + 1\\) learning_rate = 0.01 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) Train Model 1 epoch: going through the whole x_train data once 100 epochs: 100x mapping x_train = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] Process Convert inputs/labels to tensors with gradients Clear gradient buffets Get output given inputs Get loss Get gradients w.r.t. parameters Update parameters using gradients parameters = parameters - learning_rate * parameters_gradients REPEAT epochs = 100 for epoch in range ( epochs ): epoch += 1 # Convert numpy array to torch Variable inputs = torch . from_numpy ( x_train ) . requires_grad_ () labels = torch . from_numpy ( y_train ) # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward to get output outputs = model ( inputs ) # Calculate Loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () print ( 'epoch {} , loss {} ' . format ( epoch , loss . item ())) epoch 1 , loss 140.58143615722656 epoch 2 , loss 11.467253684997559 epoch 3 , loss 0.9358152747154236 epoch 4 , loss 0.07679400593042374 epoch 5 , loss 0.0067212567664682865 epoch 6 , loss 0.0010006226366385818 epoch 7 , loss 0.0005289533291943371 epoch 8 , loss 0.0004854927829001099 epoch 9 , loss 0.00047700389404781163 epoch 10 , loss 0.0004714332753792405 epoch 11 , loss 0.00046614606981165707 epoch 12 , loss 0.0004609318566508591 epoch 13 , loss 0.0004557870561257005 epoch 14 , loss 0.00045069155748933554 epoch 15 , loss 0.00044567222357727587 epoch 16 , loss 0.00044068993884138763 epoch 17 , loss 0.00043576463940553367 epoch 18 , loss 0.00043090470717288554 epoch 19 , loss 0.00042609183583408594 epoch 20 , loss 0.0004213254142086953 epoch 21 , loss 0.0004166301223449409 epoch 22 , loss 0.0004119801160413772 epoch 23 , loss 0.00040738462121225893 epoch 24 , loss 0.0004028224211651832 epoch 25 , loss 0.0003983367350883782 epoch 26 , loss 0.0003938761365134269 epoch 27 , loss 0.000389480876037851 epoch 28 , loss 0.00038514015614055097 epoch 29 , loss 0.000380824290914461 epoch 30 , loss 0.00037657516077160835 epoch 31 , loss 0.000372376263840124 epoch 32 , loss 0.0003682126116473228 epoch 33 , loss 0.0003640959912445396 epoch 34 , loss 0.00036003670538775623 epoch 35 , loss 0.00035601368290372193 epoch 36 , loss 0.00035203873994760215 epoch 37 , loss 0.00034810820943675935 epoch 38 , loss 0.000344215368386358 epoch 39 , loss 0.0003403784066904336 epoch 40 , loss 0.00033658024040050805 epoch 41 , loss 0.0003328165039420128 epoch 42 , loss 0.0003291067841928452 epoch 43 , loss 0.0003254293987993151 epoch 44 , loss 0.0003217888588551432 epoch 45 , loss 0.0003182037326041609 epoch 46 , loss 0.0003146533854305744 epoch 47 , loss 0.00031113551813177764 epoch 48 , loss 0.0003076607536058873 epoch 49 , loss 0.00030422292184084654 epoch 50 , loss 0.00030083119054324925 epoch 51 , loss 0.00029746422660537064 epoch 52 , loss 0.0002941471466328949 epoch 53 , loss 0.00029085995629429817 epoch 54 , loss 0.0002876132493838668 epoch 55 , loss 0.00028440452297218144 epoch 56 , loss 0.00028122696676291525 epoch 57 , loss 0.00027808290906250477 epoch 58 , loss 0.00027497278642840683 epoch 59 , loss 0.00027190230321139097 epoch 60 , loss 0.00026887087733484805 epoch 61 , loss 0.0002658693410921842 epoch 62 , loss 0.0002629039518069476 epoch 63 , loss 0.00025996880140155554 epoch 64 , loss 0.0002570618235040456 epoch 65 , loss 0.00025419273879379034 epoch 66 , loss 0.00025135406758636236 epoch 67 , loss 0.0002485490695107728 epoch 68 , loss 0.0002457649679854512 epoch 69 , loss 0.0002430236927466467 epoch 70 , loss 0.00024031475186347961 epoch 71 , loss 0.00023762597993481904 epoch 72 , loss 0.00023497406800743192 epoch 73 , loss 0.0002323519001947716 epoch 74 , loss 0.00022976362379267812 epoch 75 , loss 0.0002271933335578069 epoch 76 , loss 0.00022465786605607718 epoch 77 , loss 0.00022214400814846158 epoch 78 , loss 0.00021966728672850877 epoch 79 , loss 0.0002172116219298914 epoch 80 , loss 0.00021478648704942316 epoch 81 , loss 0.00021239375928416848 epoch 82 , loss 0.0002100227284245193 epoch 83 , loss 0.00020767028036061674 epoch 84 , loss 0.00020534756185952574 epoch 85 , loss 0.00020305956422816962 epoch 86 , loss 0.0002007894654525444 epoch 87 , loss 0.00019854879064951092 epoch 88 , loss 0.00019633043848443776 epoch 89 , loss 0.00019413618429098278 epoch 90 , loss 0.00019197272195015103 epoch 91 , loss 0.0001898303598864004 epoch 92 , loss 0.00018771187751553953 epoch 93 , loss 0.00018561164324637502 epoch 94 , loss 0.00018354636267758906 epoch 95 , loss 0.00018149390234611928 epoch 96 , loss 0.0001794644631445408 epoch 97 , loss 0.00017746571393217891 epoch 98 , loss 0.00017548113828524947 epoch 99 , loss 0.00017352371651213616 epoch 100 , loss 0.00017157981346827 Looking at predicted values # Purely inference predicted = model ( torch . from_numpy ( x_train ) . requires_grad_ ()) . data . numpy () predicted array ([[ 0.9756333 ], [ 2.9791424 ], [ 4.982651 ], [ 6.9861603 ], [ 8.98967 ], [ 10.993179 ], [ 12.996688 ], [ 15.000196 ], [ 17.003706 ], [ 19.007215 ], [ 21.010725 ]], dtype = float32 ) Looking at training values These are the true values, you can see how it's able to predict similar values. # y = 2x + 1 y_train array ([[ 1. ], [ 3. ], [ 5. ], [ 7. ], [ 9. ], [ 11. ], [ 13. ], [ 15. ], [ 17. ], [ 19. ], [ 21. ]], dtype = float32 ) Plot of predicted and actual values # Clear figure plt . clf () # Get predictions predicted = model ( torch . from_numpy ( x_train ) . requires_grad_ ()) . data . numpy () # Plot true data plt . plot ( x_train , y_train , 'go' , label = 'True data' , alpha = 0.5 ) # Plot predictions plt . plot ( x_train , predicted , '--' , label = 'Predictions' , alpha = 0.5 ) # Legend and plot plt . legend ( loc = 'best' ) plt . show () Save Model save_model = False if save_model is True : # Saves only parameters # alpha & beta torch . save ( model . state_dict (), 'awesome_model.pkl' ) Load Model load_model = False if load_model is True : model . load_state_dict ( torch . load ( 'awesome_model.pkl' )) Building a Linear Regression Model with PyTorch (GPU) \u00b6 CPU Summary import torch import torch.nn as nn ''' STEP 1: CREATE MODEL CLASS ''' class LinearRegressionModel ( nn . Module ): def __init__ ( self , input_dim , output_dim ): super ( LinearRegressionModel , self ) . __init__ () self . linear = nn . Linear ( input_dim , output_dim ) def forward ( self , x ): out = self . linear ( x ) return out ''' STEP 2: INSTANTIATE MODEL CLASS ''' input_dim = 1 output_dim = 1 model = LinearRegressionModel ( input_dim , output_dim ) ''' STEP 3: INSTANTIATE LOSS CLASS ''' criterion = nn . MSELoss () ''' STEP 4: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.01 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 5: TRAIN THE MODEL ''' epochs = 100 for epoch in range ( epochs ): epoch += 1 # Convert numpy array to torch Variable inputs = torch . from_numpy ( x_train ) . requires_grad_ () labels = torch . from_numpy ( y_train ) # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward to get output outputs = model ( inputs ) # Calculate Loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () GPU Summary Just remember always 2 things must be on GPU model tensors import torch import torch.nn as nn import numpy as np ''' STEP 1: CREATE MODEL CLASS ''' class LinearRegressionModel ( nn . Module ): def __init__ ( self , input_dim , output_dim ): super ( LinearRegressionModel , self ) . __init__ () self . linear = nn . Linear ( input_dim , output_dim ) def forward ( self , x ): out = self . linear ( x ) return out ''' STEP 2: INSTANTIATE MODEL CLASS ''' input_dim = 1 output_dim = 1 model = LinearRegressionModel ( input_dim , output_dim ) ####################### # USE GPU FOR MODEL # ####################### device = torch . device ( \"cuda:0\" if torch . cuda . is_available () else \"cpu\" ) model . to ( device ) ''' STEP 3: INSTANTIATE LOSS CLASS ''' criterion = nn . MSELoss () ''' STEP 4: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.01 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 5: TRAIN THE MODEL ''' epochs = 100 for epoch in range ( epochs ): epoch += 1 # Convert numpy array to torch Variable ####################### # USE GPU FOR MODEL # ####################### inputs = torch . from_numpy ( x_train ) . to ( device ) labels = torch . from_numpy ( y_train ) . to ( device ) # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward to get output outputs = model ( inputs ) # Calculate Loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () # Logging print ( 'epoch {} , loss {} ' . format ( epoch , loss . item ())) epoch 1 , loss 336.0314025878906 epoch 2 , loss 27.67657470703125 epoch 3 , loss 2.5220539569854736 epoch 4 , loss 0.46732547879219055 epoch 5 , loss 0.2968060076236725 epoch 6 , loss 0.2800087630748749 epoch 7 , loss 0.27578213810920715 epoch 8 , loss 0.2726128399372101 epoch 9 , loss 0.269561231136322 epoch 10 , loss 0.2665504515171051 epoch 11 , loss 0.2635740041732788 epoch 12 , loss 0.26063060760498047 epoch 13 , loss 0.2577202618122101 epoch 14 , loss 0.2548423111438751 epoch 15 , loss 0.25199657678604126 epoch 16 , loss 0.24918246269226074 epoch 17 , loss 0.24639996886253357 epoch 18 , loss 0.24364829063415527 epoch 19 , loss 0.24092751741409302 epoch 20 , loss 0.2382371574640274 epoch 21 , loss 0.23557686805725098 epoch 22 , loss 0.2329462170600891 epoch 23 , loss 0.2303449958562851 epoch 24 , loss 0.22777271270751953 epoch 25 , loss 0.2252292037010193 epoch 26 , loss 0.22271405160427094 epoch 27 , loss 0.22022713720798492 epoch 28 , loss 0.21776780486106873 epoch 29 , loss 0.21533599495887756 epoch 30 , loss 0.21293145418167114 epoch 31 , loss 0.21055366098880768 epoch 32 , loss 0.20820240676403046 epoch 33 , loss 0.2058774083852768 epoch 34 , loss 0.20357847213745117 epoch 35 , loss 0.20130516588687897 epoch 36 , loss 0.1990572065114975 epoch 37 , loss 0.19683438539505005 epoch 38 , loss 0.19463638961315155 epoch 39 , loss 0.19246290624141693 epoch 40 , loss 0.1903136670589447 epoch 41 , loss 0.1881885528564453 epoch 42 , loss 0.18608702719211578 epoch 43 , loss 0.18400898575782776 epoch 44 , loss 0.18195408582687378 epoch 45 , loss 0.17992223799228668 epoch 46 , loss 0.17791320383548737 epoch 47 , loss 0.17592646181583405 epoch 48 , loss 0.17396186292171478 epoch 49 , loss 0.17201924324035645 epoch 50 , loss 0.17009828984737396 epoch 51 , loss 0.16819894313812256 epoch 52 , loss 0.16632060706615448 epoch 53 , loss 0.16446338593959808 epoch 54 , loss 0.16262666881084442 epoch 55 , loss 0.16081078350543976 epoch 56 , loss 0.15901507437229156 epoch 57 , loss 0.15723931789398193 epoch 58 , loss 0.15548335015773773 epoch 59 , loss 0.15374726057052612 epoch 60 , loss 0.1520303338766098 epoch 61 , loss 0.15033268928527832 epoch 62 , loss 0.14865389466285706 epoch 63 , loss 0.14699392020702362 epoch 64 , loss 0.14535246789455414 epoch 65 , loss 0.14372935891151428 epoch 66 , loss 0.14212435483932495 epoch 67 , loss 0.14053721725940704 epoch 68 , loss 0.13896773755550385 epoch 69 , loss 0.1374160647392273 epoch 70 , loss 0.1358814686536789 epoch 71 , loss 0.13436420261859894 epoch 72 , loss 0.13286370038986206 epoch 73 , loss 0.1313801407814026 epoch 74 , loss 0.12991292774677277 epoch 75 , loss 0.12846232950687408 epoch 76 , loss 0.1270277351140976 epoch 77 , loss 0.12560924887657166 epoch 78 , loss 0.12420656532049179 epoch 79 , loss 0.12281957268714905 epoch 80 , loss 0.1214480847120285 epoch 81 , loss 0.12009195983409882 epoch 82 , loss 0.1187509223818779 epoch 83 , loss 0.11742479354143143 epoch 84 , loss 0.11611353605985641 epoch 85 , loss 0.11481687426567078 epoch 86 , loss 0.11353478580713272 epoch 87 , loss 0.11226697266101837 epoch 88 , loss 0.11101329326629639 epoch 89 , loss 0.10977360606193542 epoch 90 , loss 0.10854770988225937 epoch 91 , loss 0.10733554512262344 epoch 92 , loss 0.10613703727722168 epoch 93 , loss 0.10495180636644363 epoch 94 , loss 0.10377981513738632 epoch 95 , loss 0.10262089222669601 epoch 96 , loss 0.10147502273321152 epoch 97 , loss 0.1003417894244194 epoch 98 , loss 0.09922132641077042 epoch 99 , loss 0.0981132984161377 epoch 100 , loss 0.09701769798994064 Summary \u00b6 We've learnt to... Success Simple linear regression basics \\(y = Ax + B\\) \\(y = 2x + 1\\) Example of simple linear regression Aim of linear regression Minimizing distance between the points and the line Calculate \"distance\" through MSE Calculate gradients Update parameters with parameters = parameters - learning_rate * gradients Slowly update parameters \\(A\\) and \\(B\\) model the linear relationship between \\(y\\) and \\(x\\) of the form \\(y = 2x + 1\\) Built a linear regression model in CPU and GPU Step 1: Create Model Class Step 2: Instantiate Model Class Step 3: Instantiate Loss Class Step 4: Instantiate Optimizer Class Step 5: Train Model Important things to be on GPU model tensors with gradients How to bring to GPU ? model_name.to(device) variable_name.to(device) Citation \u00b6 If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Linear Regression"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#linear-regression-with-pytorch","text":"Run Jupyter Notebook You can run the code for this section in this jupyter notebook link .","title":"Linear Regression with PyTorch"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#about-linear-regression","text":"","title":"About Linear Regression"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#simple-linear-regression-basics","text":"Allows us to understand relationship between two continuous variables Example x: independent variable weight y: dependent variable height \\(y = \\alpha x + \\beta\\)","title":"Simple Linear Regression Basics"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#example-of-simple-linear-regression","text":"Create plot for simple linear regression Take note that this code is not important at all. It simply creates random data points and does a simple best-fit line to best approximate the underlying function if one even exists. import numpy as np import matplotlib.pyplot as plt % matplotlib inline # Creates 50 random x and y numbers np . random . seed ( 1 ) n = 50 x = np . random . randn ( n ) y = x * np . random . randn ( n ) # Makes the dots colorful colors = np . random . rand ( n ) # Plots best-fit line via polyfit plt . plot ( np . unique ( x ), np . poly1d ( np . polyfit ( x , y , 1 ))( np . unique ( x ))) # Plots the random x and y data points we created # Interestingly, alpha makes it more aesthetically pleasing plt . scatter ( x , y , c = colors , alpha = 0.5 ) plt . show ()","title":"Example of simple linear regression"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#aim-of-linear-regression","text":"Minimize the distance between the points and the line ( \\(y = \\alpha x + \\beta\\) ) Adjusting Coefficient: \\(\\alpha\\) Bias/intercept: \\(\\beta\\)","title":"Aim of Linear Regression"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#building-a-linear-regression-model-with-pytorch","text":"","title":"Building a Linear Regression Model with PyTorch"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#example","text":"Coefficient: \\(\\alpha = 2\\) Bias/intercept: \\(\\beta = 1\\) Equation: \\(y = 2x + 1\\)","title":"Example"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#building-a-toy-dataset","text":"Create a list of values from 0 to 11 x_values = [ i for i in range ( 11 )] x_values [ 0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 ] Convert list of numbers to numpy array # Convert to numpy x_train = np . array ( x_values , dtype = np . float32 ) x_train . shape ( 11 ,) Convert to 2-dimensional array If you don't this you will get an error stating you need 2D. Simply just reshape accordingly if you ever face such errors down the road. # IMPORTANT: 2D required x_train = x_train . reshape ( - 1 , 1 ) x_train . shape ( 11 , 1 ) Create list of y values We want y values for every x value we have above. \\(y = 2x + 1\\) y_values = [ 2 * i + 1 for i in x_values ] y_values [ 1 , 3 , 5 , 7 , 9 , 11 , 13 , 15 , 17 , 19 , 21 ] Alternative to create list of y values If you're weak in list iterators, this might be an easier alternative. # In case you're weak in list iterators... y_values = [] for i in x_values : result = 2 * i + 1 y_values . append ( result ) y_values [ 1 , 3 , 5 , 7 , 9 , 11 , 13 , 15 , 17 , 19 , 21 ] Convert to numpy array You will slowly get a hang on how when you deal with PyTorch tensors, you just keep on making sure your raw data is in numpy form to make sure everything's good. y_train = np . array ( y_values , dtype = np . float32 ) y_train . shape ( 11 ,) Reshape y numpy array to 2-dimension # IMPORTANT: 2D required y_train = y_train . reshape ( - 1 , 1 ) y_train . shape ( 11 , 1 )","title":"Building a Toy Dataset"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#building-model","text":"Critical Imports import torch import torch.nn as nn Create Model Linear model True Equation: \\(y = 2x + 1\\) Forward Example Input \\(x = 1\\) Output \\(\\hat y = ?\\) # Create class class LinearRegressionModel ( nn . Module ): def __init__ ( self , input_dim , output_dim ): super ( LinearRegressionModel , self ) . __init__ () self . linear = nn . Linear ( input_dim , output_dim ) def forward ( self , x ): out = self . linear ( x ) return out Instantiate Model Class input: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] desired output: [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21] input_dim = 1 output_dim = 1 model = LinearRegressionModel ( input_dim , output_dim ) Instantiate Loss Class MSE Loss: Mean Squared Error \\(MSE = \\frac{1}{n} \\sum_{i=1}^n(\\hat y_i - y_i)^2\\) \\(\\hat y\\) : prediction \\(y\\) : true value criterion = nn . MSELoss () Instantiate Optimizer Class Simplified equation \\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\) \\(\\theta\\) : parameters (our variables) \\(\\eta\\) : learning rate (how fast we want to learn) \\(\\nabla_\\theta\\) : parameters' gradients Even simplier equation parameters = parameters - learning_rate * parameters_gradients parameters: \\(\\alpha\\) and \\(\\beta\\) in \\(y = \\alpha x + \\beta\\) desired parameters: \\(\\alpha = 2\\) and \\(\\beta = 1\\) in \\(y = 2x + 1\\) learning_rate = 0.01 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) Train Model 1 epoch: going through the whole x_train data once 100 epochs: 100x mapping x_train = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] Process Convert inputs/labels to tensors with gradients Clear gradient buffets Get output given inputs Get loss Get gradients w.r.t. parameters Update parameters using gradients parameters = parameters - learning_rate * parameters_gradients REPEAT epochs = 100 for epoch in range ( epochs ): epoch += 1 # Convert numpy array to torch Variable inputs = torch . from_numpy ( x_train ) . requires_grad_ () labels = torch . from_numpy ( y_train ) # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward to get output outputs = model ( inputs ) # Calculate Loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () print ( 'epoch {} , loss {} ' . format ( epoch , loss . item ())) epoch 1 , loss 140.58143615722656 epoch 2 , loss 11.467253684997559 epoch 3 , loss 0.9358152747154236 epoch 4 , loss 0.07679400593042374 epoch 5 , loss 0.0067212567664682865 epoch 6 , loss 0.0010006226366385818 epoch 7 , loss 0.0005289533291943371 epoch 8 , loss 0.0004854927829001099 epoch 9 , loss 0.00047700389404781163 epoch 10 , loss 0.0004714332753792405 epoch 11 , loss 0.00046614606981165707 epoch 12 , loss 0.0004609318566508591 epoch 13 , loss 0.0004557870561257005 epoch 14 , loss 0.00045069155748933554 epoch 15 , loss 0.00044567222357727587 epoch 16 , loss 0.00044068993884138763 epoch 17 , loss 0.00043576463940553367 epoch 18 , loss 0.00043090470717288554 epoch 19 , loss 0.00042609183583408594 epoch 20 , loss 0.0004213254142086953 epoch 21 , loss 0.0004166301223449409 epoch 22 , loss 0.0004119801160413772 epoch 23 , loss 0.00040738462121225893 epoch 24 , loss 0.0004028224211651832 epoch 25 , loss 0.0003983367350883782 epoch 26 , loss 0.0003938761365134269 epoch 27 , loss 0.000389480876037851 epoch 28 , loss 0.00038514015614055097 epoch 29 , loss 0.000380824290914461 epoch 30 , loss 0.00037657516077160835 epoch 31 , loss 0.000372376263840124 epoch 32 , loss 0.0003682126116473228 epoch 33 , loss 0.0003640959912445396 epoch 34 , loss 0.00036003670538775623 epoch 35 , loss 0.00035601368290372193 epoch 36 , loss 0.00035203873994760215 epoch 37 , loss 0.00034810820943675935 epoch 38 , loss 0.000344215368386358 epoch 39 , loss 0.0003403784066904336 epoch 40 , loss 0.00033658024040050805 epoch 41 , loss 0.0003328165039420128 epoch 42 , loss 0.0003291067841928452 epoch 43 , loss 0.0003254293987993151 epoch 44 , loss 0.0003217888588551432 epoch 45 , loss 0.0003182037326041609 epoch 46 , loss 0.0003146533854305744 epoch 47 , loss 0.00031113551813177764 epoch 48 , loss 0.0003076607536058873 epoch 49 , loss 0.00030422292184084654 epoch 50 , loss 0.00030083119054324925 epoch 51 , loss 0.00029746422660537064 epoch 52 , loss 0.0002941471466328949 epoch 53 , loss 0.00029085995629429817 epoch 54 , loss 0.0002876132493838668 epoch 55 , loss 0.00028440452297218144 epoch 56 , loss 0.00028122696676291525 epoch 57 , loss 0.00027808290906250477 epoch 58 , loss 0.00027497278642840683 epoch 59 , loss 0.00027190230321139097 epoch 60 , loss 0.00026887087733484805 epoch 61 , loss 0.0002658693410921842 epoch 62 , loss 0.0002629039518069476 epoch 63 , loss 0.00025996880140155554 epoch 64 , loss 0.0002570618235040456 epoch 65 , loss 0.00025419273879379034 epoch 66 , loss 0.00025135406758636236 epoch 67 , loss 0.0002485490695107728 epoch 68 , loss 0.0002457649679854512 epoch 69 , loss 0.0002430236927466467 epoch 70 , loss 0.00024031475186347961 epoch 71 , loss 0.00023762597993481904 epoch 72 , loss 0.00023497406800743192 epoch 73 , loss 0.0002323519001947716 epoch 74 , loss 0.00022976362379267812 epoch 75 , loss 0.0002271933335578069 epoch 76 , loss 0.00022465786605607718 epoch 77 , loss 0.00022214400814846158 epoch 78 , loss 0.00021966728672850877 epoch 79 , loss 0.0002172116219298914 epoch 80 , loss 0.00021478648704942316 epoch 81 , loss 0.00021239375928416848 epoch 82 , loss 0.0002100227284245193 epoch 83 , loss 0.00020767028036061674 epoch 84 , loss 0.00020534756185952574 epoch 85 , loss 0.00020305956422816962 epoch 86 , loss 0.0002007894654525444 epoch 87 , loss 0.00019854879064951092 epoch 88 , loss 0.00019633043848443776 epoch 89 , loss 0.00019413618429098278 epoch 90 , loss 0.00019197272195015103 epoch 91 , loss 0.0001898303598864004 epoch 92 , loss 0.00018771187751553953 epoch 93 , loss 0.00018561164324637502 epoch 94 , loss 0.00018354636267758906 epoch 95 , loss 0.00018149390234611928 epoch 96 , loss 0.0001794644631445408 epoch 97 , loss 0.00017746571393217891 epoch 98 , loss 0.00017548113828524947 epoch 99 , loss 0.00017352371651213616 epoch 100 , loss 0.00017157981346827 Looking at predicted values # Purely inference predicted = model ( torch . from_numpy ( x_train ) . requires_grad_ ()) . data . numpy () predicted array ([[ 0.9756333 ], [ 2.9791424 ], [ 4.982651 ], [ 6.9861603 ], [ 8.98967 ], [ 10.993179 ], [ 12.996688 ], [ 15.000196 ], [ 17.003706 ], [ 19.007215 ], [ 21.010725 ]], dtype = float32 ) Looking at training values These are the true values, you can see how it's able to predict similar values. # y = 2x + 1 y_train array ([[ 1. ], [ 3. ], [ 5. ], [ 7. ], [ 9. ], [ 11. ], [ 13. ], [ 15. ], [ 17. ], [ 19. ], [ 21. ]], dtype = float32 ) Plot of predicted and actual values # Clear figure plt . clf () # Get predictions predicted = model ( torch . from_numpy ( x_train ) . requires_grad_ ()) . data . numpy () # Plot true data plt . plot ( x_train , y_train , 'go' , label = 'True data' , alpha = 0.5 ) # Plot predictions plt . plot ( x_train , predicted , '--' , label = 'Predictions' , alpha = 0.5 ) # Legend and plot plt . legend ( loc = 'best' ) plt . show () Save Model save_model = False if save_model is True : # Saves only parameters # alpha & beta torch . save ( model . state_dict (), 'awesome_model.pkl' ) Load Model load_model = False if load_model is True : model . load_state_dict ( torch . load ( 'awesome_model.pkl' ))","title":"Building Model"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#building-a-linear-regression-model-with-pytorch-gpu","text":"CPU Summary import torch import torch.nn as nn ''' STEP 1: CREATE MODEL CLASS ''' class LinearRegressionModel ( nn . Module ): def __init__ ( self , input_dim , output_dim ): super ( LinearRegressionModel , self ) . __init__ () self . linear = nn . Linear ( input_dim , output_dim ) def forward ( self , x ): out = self . linear ( x ) return out ''' STEP 2: INSTANTIATE MODEL CLASS ''' input_dim = 1 output_dim = 1 model = LinearRegressionModel ( input_dim , output_dim ) ''' STEP 3: INSTANTIATE LOSS CLASS ''' criterion = nn . MSELoss () ''' STEP 4: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.01 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 5: TRAIN THE MODEL ''' epochs = 100 for epoch in range ( epochs ): epoch += 1 # Convert numpy array to torch Variable inputs = torch . from_numpy ( x_train ) . requires_grad_ () labels = torch . from_numpy ( y_train ) # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward to get output outputs = model ( inputs ) # Calculate Loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () GPU Summary Just remember always 2 things must be on GPU model tensors import torch import torch.nn as nn import numpy as np ''' STEP 1: CREATE MODEL CLASS ''' class LinearRegressionModel ( nn . Module ): def __init__ ( self , input_dim , output_dim ): super ( LinearRegressionModel , self ) . __init__ () self . linear = nn . Linear ( input_dim , output_dim ) def forward ( self , x ): out = self . linear ( x ) return out ''' STEP 2: INSTANTIATE MODEL CLASS ''' input_dim = 1 output_dim = 1 model = LinearRegressionModel ( input_dim , output_dim ) ####################### # USE GPU FOR MODEL # ####################### device = torch . device ( \"cuda:0\" if torch . cuda . is_available () else \"cpu\" ) model . to ( device ) ''' STEP 3: INSTANTIATE LOSS CLASS ''' criterion = nn . MSELoss () ''' STEP 4: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.01 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 5: TRAIN THE MODEL ''' epochs = 100 for epoch in range ( epochs ): epoch += 1 # Convert numpy array to torch Variable ####################### # USE GPU FOR MODEL # ####################### inputs = torch . from_numpy ( x_train ) . to ( device ) labels = torch . from_numpy ( y_train ) . to ( device ) # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward to get output outputs = model ( inputs ) # Calculate Loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () # Logging print ( 'epoch {} , loss {} ' . format ( epoch , loss . item ())) epoch 1 , loss 336.0314025878906 epoch 2 , loss 27.67657470703125 epoch 3 , loss 2.5220539569854736 epoch 4 , loss 0.46732547879219055 epoch 5 , loss 0.2968060076236725 epoch 6 , loss 0.2800087630748749 epoch 7 , loss 0.27578213810920715 epoch 8 , loss 0.2726128399372101 epoch 9 , loss 0.269561231136322 epoch 10 , loss 0.2665504515171051 epoch 11 , loss 0.2635740041732788 epoch 12 , loss 0.26063060760498047 epoch 13 , loss 0.2577202618122101 epoch 14 , loss 0.2548423111438751 epoch 15 , loss 0.25199657678604126 epoch 16 , loss 0.24918246269226074 epoch 17 , loss 0.24639996886253357 epoch 18 , loss 0.24364829063415527 epoch 19 , loss 0.24092751741409302 epoch 20 , loss 0.2382371574640274 epoch 21 , loss 0.23557686805725098 epoch 22 , loss 0.2329462170600891 epoch 23 , loss 0.2303449958562851 epoch 24 , loss 0.22777271270751953 epoch 25 , loss 0.2252292037010193 epoch 26 , loss 0.22271405160427094 epoch 27 , loss 0.22022713720798492 epoch 28 , loss 0.21776780486106873 epoch 29 , loss 0.21533599495887756 epoch 30 , loss 0.21293145418167114 epoch 31 , loss 0.21055366098880768 epoch 32 , loss 0.20820240676403046 epoch 33 , loss 0.2058774083852768 epoch 34 , loss 0.20357847213745117 epoch 35 , loss 0.20130516588687897 epoch 36 , loss 0.1990572065114975 epoch 37 , loss 0.19683438539505005 epoch 38 , loss 0.19463638961315155 epoch 39 , loss 0.19246290624141693 epoch 40 , loss 0.1903136670589447 epoch 41 , loss 0.1881885528564453 epoch 42 , loss 0.18608702719211578 epoch 43 , loss 0.18400898575782776 epoch 44 , loss 0.18195408582687378 epoch 45 , loss 0.17992223799228668 epoch 46 , loss 0.17791320383548737 epoch 47 , loss 0.17592646181583405 epoch 48 , loss 0.17396186292171478 epoch 49 , loss 0.17201924324035645 epoch 50 , loss 0.17009828984737396 epoch 51 , loss 0.16819894313812256 epoch 52 , loss 0.16632060706615448 epoch 53 , loss 0.16446338593959808 epoch 54 , loss 0.16262666881084442 epoch 55 , loss 0.16081078350543976 epoch 56 , loss 0.15901507437229156 epoch 57 , loss 0.15723931789398193 epoch 58 , loss 0.15548335015773773 epoch 59 , loss 0.15374726057052612 epoch 60 , loss 0.1520303338766098 epoch 61 , loss 0.15033268928527832 epoch 62 , loss 0.14865389466285706 epoch 63 , loss 0.14699392020702362 epoch 64 , loss 0.14535246789455414 epoch 65 , loss 0.14372935891151428 epoch 66 , loss 0.14212435483932495 epoch 67 , loss 0.14053721725940704 epoch 68 , loss 0.13896773755550385 epoch 69 , loss 0.1374160647392273 epoch 70 , loss 0.1358814686536789 epoch 71 , loss 0.13436420261859894 epoch 72 , loss 0.13286370038986206 epoch 73 , loss 0.1313801407814026 epoch 74 , loss 0.12991292774677277 epoch 75 , loss 0.12846232950687408 epoch 76 , loss 0.1270277351140976 epoch 77 , loss 0.12560924887657166 epoch 78 , loss 0.12420656532049179 epoch 79 , loss 0.12281957268714905 epoch 80 , loss 0.1214480847120285 epoch 81 , loss 0.12009195983409882 epoch 82 , loss 0.1187509223818779 epoch 83 , loss 0.11742479354143143 epoch 84 , loss 0.11611353605985641 epoch 85 , loss 0.11481687426567078 epoch 86 , loss 0.11353478580713272 epoch 87 , loss 0.11226697266101837 epoch 88 , loss 0.11101329326629639 epoch 89 , loss 0.10977360606193542 epoch 90 , loss 0.10854770988225937 epoch 91 , loss 0.10733554512262344 epoch 92 , loss 0.10613703727722168 epoch 93 , loss 0.10495180636644363 epoch 94 , loss 0.10377981513738632 epoch 95 , loss 0.10262089222669601 epoch 96 , loss 0.10147502273321152 epoch 97 , loss 0.1003417894244194 epoch 98 , loss 0.09922132641077042 epoch 99 , loss 0.0981132984161377 epoch 100 , loss 0.09701769798994064","title":"Building a Linear Regression Model with PyTorch (GPU)"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#summary","text":"We've learnt to... Success Simple linear regression basics \\(y = Ax + B\\) \\(y = 2x + 1\\) Example of simple linear regression Aim of linear regression Minimizing distance between the points and the line Calculate \"distance\" through MSE Calculate gradients Update parameters with parameters = parameters - learning_rate * gradients Slowly update parameters \\(A\\) and \\(B\\) model the linear relationship between \\(y\\) and \\(x\\) of the form \\(y = 2x + 1\\) Built a linear regression model in CPU and GPU Step 1: Create Model Class Step 2: Instantiate Model Class Step 3: Instantiate Loss Class Step 4: Instantiate Optimizer Class Step 5: Train Model Important things to be on GPU model tensors with gradients How to bring to GPU ? model_name.to(device) variable_name.to(device)","title":"Summary"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#citation","text":"If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Citation"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/","text":"Logistic Regression with PyTorch \u00b6 Run Jupyter Notebook You can run the code for this section in this jupyter notebook link . About Logistic Regression \u00b6 Logistic Regression Basics \u00b6 Classification algorithm \u00b6 Example: Spam vs No Spam Input: Bunch of words Output: Probability spam or not Basic Comparison \u00b6 Linear regression Output: numeric value given inputs Logistic regression : Output: probability [0, 1] given input belonging to a class Input/Output Comparison \u00b6 Linear regression: Multiplication Input: [1] Output: 2 Input: [2] Output: 4 Trying to model the relationship y = 2x Logistic regression: Spam Input: \"Sign up to get 1 million dollars by tonight\" Output: p = 0.8 Input: \"This is a receipt for your recent purchase with Amazon\" Output: p = 0.3 p: probability it is spam Problems of Linear Regression \u00b6 Example Fever Input : temperature Output : fever or no fever Remember Linear regression : minimize error between points and line Linear Regression Problem 1: Fever value can go negative (below 0) and positive (above 1) If you simply tried to do a simple linear regression on this fever problem, you would realize an apparent error. Fever can go beyond 1 and below 0 which does not make sense in this context. import numpy as np import matplotlib.pyplot as plt % matplotlib inline x = [ 1 , 5 , 10 , 10 , 25 , 50 , 70 , 75 , 100 ,] y = [ 0 , 0 , 0 , 0 , 0 , 1 , 1 , 1 , 1 ] colors = np . random . rand ( len ( x )) plt . plot ( np . unique ( x ), np . poly1d ( np . polyfit ( x , y , 1 ))( np . unique ( x ))) plt . ylabel ( \"Fever\" ) plt . xlabel ( \"Temperature\" ) plt . scatter ( x , y , c = colors , alpha = 0.5 ) plt . show () Linear Regression Problem 2: Fever points are not predicted with the presence of outliers Previously at least some points could be properly predicted. However, with the presence of outliers, everything goes wonky for simple linear regression, having no predictive capacity at all. import numpy as np import matplotlib.pyplot as plt x = [ 1 , 5 , 10 , 10 , 25 , 50 , 70 , 75 , 300 ] y = [ 0 , 0 , 0 , 0 , 0 , 1 , 1 , 1 , 1 ] colors = np . random . rand ( len ( x )) plt . plot ( np . unique ( x ), np . poly1d ( np . polyfit ( x , y , 1 ))( np . unique ( x ))) plt . ylabel ( \"Fever\" ) plt . xlabel ( \"Temperature\" ) plt . scatter ( x , y , c = colors , alpha = 0.5 ) plt . show () Logistic Regression In-Depth \u00b6 Predicting Probability \u00b6 Linear regression doesn't work Instead of predicting direct values: predict probability Logistic Function g() \u00b6 \"Two-class logistic regression\" \\(\\boldsymbol{y} = A\\boldsymbol{x} + \\boldsymbol{b}\\) Where \\(\\boldsymbol{y}\\) is a vector comprising the 2-class prediction \\(y_0\\) and \\(y_1\\) Where the labels are \\(y_0 = 0\\) and \\(y_1 = 1\\) Also, it's bolded because it's a vector, not a matrix. \\(g(y_1) = \\frac {1} {1 + e^{-y_1}}\\) \\(g(y_1)\\) = Estimated probability that \\(y = 1\\) \\(g(y_0) = 1 - g(y_1)\\) \\(g(y_0)\\) = Estimated probability that \\(y = 0\\) For our illustration above, we have 4 classes, so we have to use softmax function explained below Softmax Function g() \u00b6 \"Multi-class logistic regression\" Generalization of logistic function, where you can derive back to the logistic function if you've a 2 class classification problem Here, we will use a 4 class example (K = 4) as shown above to be very clear in how it relates back to that simple examaple. \\(\\boldsymbol{y} = A\\boldsymbol{x} + \\boldsymbol{b}\\) Where \\(\\boldsymbol{y}\\) is a vector comprising the 4-class prediction \\(y_0, y_1, y_2, y_3\\) Where the 4 labels (K = 4) are \\(y_0 = 0, y_1 = 1, y_2 = 2, y_3 = 3\\) \\(g(y_i) = \\frac {e^{y_i} } {\\sum^K_i e^{y_i}}\\) where K = 4 because we have 4 classes To put numbers to this equation in relation to the illustration above where we've \\(y_0 = 1.3, y_1 = 1.2, y = 4.5, y = 4.8\\) \\(g(y_0) = \\frac {e^{1.3}}{e^{1.3} + e^{1.2} + e^{4.5} + e^{4.8}} = 0.017\\) \\(g(y_1) = \\frac {e^{1.2}}{e^{1.3} + e^{1.2} + e^{4.5} + e^{4.8}} = 0.015\\) \\(g(y_2) = \\frac {e^{4.5}}{e^{1.3} + e^{1.2} + e^{4.5} + e^{4.8}} = 0.412\\) \\(g(y_3) = \\frac {e^{4.8}}{e^{1.3} + e^{1.2} + e^{4.5} + e^{4.8}} = 0.556\\) \\(g(y_0) + g(y_1) + g(y_2) + g(y_3) = 1.0\\) All softmax outputs have to sum to one as they represent a probability distribution over K classes. Take note how these numbers are not exactly as in the illustration in the softmax box but the concept is important (intentionally made so). \\(y_0\\) and \\(y_1\\) are approximately similar in values and they return similar probabilities. Similarly, \\(y_2\\) and \\(y_3\\) are approximately similar in values and they return similar probabilities. Softmax versus Soft(arg)max Do you know many researchers and anyone in deep learning in general use the term softmax when it should be soft(arg)max. This is because soft(arg)max returns the probability distribution over K classes, a vector. However, softmax only returns the max! This means you will be getting a scalar value versus a probability distribution. According to my friend, Alfredo Canziani (postdoc in NYU under Yann Lecun), it was actually a mistake made in the original paper previously but it was too late because the term softmax was adopted. Full credits to him for this tip. Cross Entropy Function D() for 2 Class \u00b6 Take note that here, \\(S\\) is our softmax outputs and \\(L\\) are our labels \\(D(S, L) = -(L log S + (1-L)log(1-S))\\) If L = 0 (label) \\(D(S, 0) = - log(1-S)\\) \\(- log(1-S)\\) : less positive if \\(S \\longrightarrow 0\\) \\(- log(1-S)\\) : more positive if \\(S \\longrightarrow 1\\) (BIGGER LOSS) If L = 1 (label) \\(D(S, 1) = - log S\\) \\(-log(S)\\) : less positive if \\(S \\longrightarrow 1\\) \\(-log(S)\\) : more positive if \\(S \\longrightarrow 0\\) (BIGGER LOSS) Numerical example of bigger or small loss You get a small error of 1e-5 if your label = 0 and your S is closer to 0 (very correct prediction). import math print ( - math . log ( 1 - 0.00001 )) You get a large error of 11.51 if your label is 0 and S is near to 1 (very wrong prediction). print ( - math . log ( 1 - 0.99999 )) You get a small error of -1e-5 if your label is 1 and S is near 1 (very correct prediction). print ( - math . log ( 0.99999 )) You get a big error of -11.51 if your label is 1 and S is near 0 (very wrong prediction). print ( - math . log ( 0.00001 )) 1.0000050000287824e-05 11.51292546497478 1.0000050000287824e-05 11.512925464970229 Cross Entropy Function D() for More Than 2 Class \u00b6 For the case where we have more than 2 class, we need a more generalized function \\(D(S, L) = - \\sum^K_1 L_i log(S_i)\\) \\(K\\) : number of classes \\(L_i\\) : label of i-th class, 1 if that's the class else 0 \\(S_i\\) : output of softmax for i-th class Cross Entropy Loss over N samples \u00b6 Goal: Minimizing Cross Entropy Loss, L \\(Loss = \\frac {1}{N} \\sum_j^N D_j\\) \\(D_j\\) : j-th sample of cross entropy function \\(D(S, L)\\) \\(N\\) : number of samples \\(Loss\\) : average cross entropy loss over N samples Building a Logistic Regression Model with PyTorch \u00b6 Steps \u00b6 Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model Step 1a: Loading MNIST Train Dataset \u00b6 Images from 1 to 9 Inspect length of training dataset You can easily load MNIST dataset with PyTorch. Here we inspect the training set, where our algorithms will learn from, and you will discover it is made up of 60,000 images. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) len ( train_dataset ) 60000 Inspecting a single image So this is how a single image is represented in numbers. It's actually a 28 pixel x 28 pixel image which is why you would end up with this 28x28 matrix of numbers. train_dataset [ 0 ] ( tensor ([[[ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0118 , 0.0706 , 0.0706 , 0.0706 , 0.4941 , 0.5333 , 0.6863 , 0.1020 , 0.6510 , 1.0000 , 0.9686 , 0.4980 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.1176 , 0.1412 , 0.3686 , 0.6039 , 0.6667 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.8824 , 0.6745 , 0.9922 , 0.9490 , 0.7647 , 0.2510 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.1922 , 0.9333 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.9843 , 0.3647 , 0.3216 , 0.3216 , 0.2196 , 0.1529 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0706 , 0.8588 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.7765 , 0.7137 , 0.9686 , 0.9451 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.3137 , 0.6118 , 0.4196 , 0.9922 , 0.9922 , 0.8039 , 0.0431 , 0.0000 , 0.1686 , 0.6039 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0549 , 0.0039 , 0.6039 , 0.9922 , 0.3529 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.5451 , 0.9922 , 0.7451 , 0.0078 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0431 , 0.7451 , 0.9922 , 0.2745 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.1373 , 0.9451 , 0.8824 , 0.6275 , 0.4235 , 0.0039 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.3176 , 0.9412 , 0.9922 , 0.9922 , 0.4667 , 0.0980 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.1765 , 0.7294 , 0.9922 , 0.9922 , 0.5882 , 0.1059 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0627 , 0.3647 , 0.9882 , 0.9922 , 0.7333 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.9765 , 0.9922 , 0.9765 , 0.2510 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.1804 , 0.5098 , 0.7176 , 0.9922 , 0.9922 , 0.8118 , 0.0078 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.1529 , 0.5804 , 0.8980 , 0.9922 , 0.9922 , 0.9922 , 0.9804 , 0.7137 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0941 , 0.4471 , 0.8667 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.7882 , 0.3059 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0902 , 0.2588 , 0.8353 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.7765 , 0.3176 , 0.0078 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0706 , 0.6706 , 0.8588 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.7647 , 0.3137 , 0.0353 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.2157 , 0.6745 , 0.8863 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.9569 , 0.5216 , 0.0431 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.5333 , 0.9922 , 0.9922 , 0.9922 , 0.8314 , 0.5294 , 0.5176 , 0.0627 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ]]]), tensor ( 5 )) Inspecting a single data point in the training dataset When you load MNIST dataset, each data point is actually a tuple containing the image matrix and the label. type ( train_dataset [ 0 ]) tuple Inspecting training dataset first element of tuple This means to access the image, you need to access the first element in the tuple. # Input Matrix train_dataset [ 0 ][ 0 ] . size () # A 28x28 sized image of a digit torch . Size ([ 1 , 28 , 28 ]) Inspecting training dataset second element of tuple The second element actually represents the image's label. Meaning if the second element says 5, it means the 28x28 matrix of numbers represent a digit 5. # Label train_dataset [ 0 ][ 1 ] tensor ( 5 ) Displaying MNIST \u00b6 Verifying shape of MNIST image As mentioned, a single MNIST image is of the shape 28 pixel x 28 pixel. import matplotlib.pyplot as plt % matplotlib inline import numpy as np train_dataset [ 0 ][ 0 ] . numpy () . shape ( 1 , 28 , 28 ) Plot image of MNIST image show_img = train_dataset [ 0 ][ 0 ] . numpy () . reshape ( 28 , 28 ) plt . imshow ( show_img , cmap = 'gray' ) Second element of tuple shows label As you would expect, the label is 5. # Label train_dataset [ 0 ][ 1 ] tensor ( 5 ) Plot second image of MNIST image show_img = train_dataset [ 1 ][ 0 ] . numpy () . reshape ( 28 , 28 ) plt . imshow ( show_img , cmap = 'gray' ) Second element of tuple shows label We should see 0 here as the label. # Label train_dataset [ 1 ][ 1 ] tensor ( 0 ) Step 1b: Loading MNIST Test Dataset \u00b6 Show our algorithm works beyond the data we have trained on. Out-of-sample Load test dataset Compared to the 60k images in the training set, the testing set where the model will not be trained on has 10k images to check for its out-of-sample performance. test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) len ( test_dataset ) 10000 Test dataset elements Exactly like the training set, the testing set has 10k tuples containing the 28x28 matrices and their respective labels. type ( test_dataset [ 0 ]) tuple Test dataset first element in tuple This contains the image matrix, similar to the training set. # Image matrix test_dataset [ 0 ][ 0 ] . size () torch . Size ([ 1 , 28 , 28 ]) Plot image sample from test dataset show_img = test_dataset [ 0 ][ 0 ] . numpy () . reshape ( 28 , 28 ) plt . imshow ( show_img , cmap = 'gray' ) Test dataset second element in tuple # Label test_dataset [ 0 ][ 1 ] tensor ( 7 ) Step 2: Make Dataset Iterable \u00b6 Aim: make the dataset iterable totaldata : 60000 minibatch : 100 Number of examples in 1 iteration iterations : 3000 1 iteration: one mini-batch forward & backward pass epochs 1 epoch: running through the whole dataset once \\(epochs = iterations \\div \\frac{totaldata}{minibatch} = 3000 \\div \\frac{60000}{100} = 5\\) Recap training dataset Remember training dataset has 60k images and testing dataset has 10k images. len ( train_dataset ) 60000 Defining epochs When the model goes through the whole 60k images once, learning how to classify 0-9, it's consider 1 epoch. However, there's a concept of batch size where it means the model would look at 100 images before updating the model's weights, thereby learning. When the model updates its weights (parameters) after looking at all the images, this is considered 1 iteration. batch_size = 100 We arbitrarily set 3000 iterations here which means the model would update 3000 times. n_iters = 3000 One epoch consists of 60,000 / 100 = 600 iterations. Because we would like to go through 3000 iterations, this implies we would have 3000 / 600 = 5 epochs as each epoch has 600 iterations. num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) num_epochs 5 Create Iterable Object: Training Dataset train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) Check Iterability import collections isinstance ( train_loader , collections . Iterable ) True Create Iterable Object: Testing Dataset # Iterable object test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) Check iterability of testing dataset isinstance ( test_loader , collections . Iterable ) True Iterate through dataset This is just a simplified example of what we're doing above where we're creating an iterable object lst to loop through so we can access all the images img_1 and img_2 . Above, the equivalent of lst is train_loader and test_loader . img_1 = np . ones (( 28 , 28 )) img_2 = np . ones (( 28 , 28 )) lst = [ img_1 , img_2 ] # Need to iterate # Think of numbers as the images for i in lst : print ( i . shape ) ( 28 , 28 ) ( 28 , 28 ) Step 3: Building Model \u00b6 Create model class # Same as linear regression! class LogisticRegressionModel ( nn . Module ): def __init__ ( self , input_dim , output_dim ): super ( LogisticRegressionModel , self ) . __init__ () self . linear = nn . Linear ( input_dim , output_dim ) def forward ( self , x ): out = self . linear ( x ) return out Step 4: Instantiate Model Class \u00b6 Input dimension: Size of image \\(28 \\times 28 = 784\\) Output dimension: 10 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 Check size of dataset This should be 28x28. # Size of images train_dataset [ 0 ][ 0 ] . size () torch . Size ([ 1 , 28 , 28 ]) Instantiate model class based on input and out dimensions As we're trying to classify digits 0-9 a total of 10 classes, our output dimension is 10. And we're feeding the model with 28x28 images, hence our input dimension is 28x28. input_dim = 28 * 28 output_dim = 10 model = LogisticRegressionModel ( input_dim , output_dim ) Step 5: Instantiate Loss Class \u00b6 Logistic Regression : Cross Entropy Loss Linear Regression: MSE Create Cross Entry Loss Class Unlike linear regression, we do not use MSE here, we need Cross Entry Loss to calculate our loss before we backpropagate and update our parameters. criterion = nn . CrossEntropyLoss () What happens in nn.CrossEntropyLoss()? It does 2 things at the same time. 1. Computes softmax (logistic/softmax function) 2. Computes cross entropy Step 6: Instantiate Optimizer Class \u00b6 Simplified equation \\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\) \\(\\theta\\) : parameters (our variables) \\(\\eta\\) : learning rate (how fast we want to learn) \\(\\nabla_\\theta\\) : parameters' gradients Even simplier equation parameters = parameters - learning_rate * parameters_gradients At every iteration, we update our model's parameters Create optimizer Similar to what we've covered above, this calculates the parameters' gradients and update them subsequently. learning_rate = 0.001 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) Parameters In-Depth You'll realize we have 2 sets of parameters, 10x784 which is A and 10x1 which is b in the \\(y = AX + b\\) equation where X is our input of size 784. We'll go into details subsequently how these parameters interact with our input to produce our 10x1 output. # Type of parameter object print ( model . parameters ()) # Length of parameters print ( len ( list ( model . parameters ()))) # FC 1 Parameters print ( list ( model . parameters ())[ 0 ] . size ()) # FC 1 Bias Parameters print ( list ( model . parameters ())[ 1 ] . size ()) < generator object Module . parameters at 0x7ff7c884f830 > 2 torch . Size ([ 10 , 784 ]) torch . Size ([ 10 ]) Quick Matrix Product Review Example 1: matrix product \\(A: (100, 10)\\) \\(B: (10, 1)\\) \\(A \\cdot B = (100, 10) \\cdot (10, 1) = (100, 1)\\) Example 2: matrix product \\(A: (50, 5)\\) \\(B: (5, 2)\\) \\(A \\cdot B = (50, 5) \\cdot (5, 2) = (50, 2)\\) Example 3: element-wise addition \\(A: (10, 1)\\) \\(B: (10, 1)\\) \\(A + B = (10, 1)\\) Step 7: Train Model \u00b6 7 step process for training models Process Convert inputs/labels to tensors with gradients Clear gradient buffets Get output given inputs Get loss Get gradients w.r.t. parameters Update parameters using gradients parameters = parameters - learning_rate * parameters_gradients REPEAT iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () labels = labels # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 1.8513233661651611 . Accuracy : 70 Iteration : 1000. Loss : 1.5732524394989014 . Accuracy : 77 Iteration : 1500. Loss : 1.3840199708938599 . Accuracy : 79 Iteration : 2000. Loss : 1.1711134910583496 . Accuracy : 81 Iteration : 2500. Loss : 1.1094708442687988 . Accuracy : 82 Iteration : 3000. Loss : 1.002761721611023 . Accuracy : 82 Break Down Accuracy Calculation \u00b6 Printing outputs of our model As we've trained our model, we can extract the accuracy calculation portion to understand what's happening without re-training the model. This would print out the output of the model's predictions on your notebook. iter_test = 0 for images , labels in test_loader : iter_test += 1 images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () outputs = model ( images ) if iter_test == 1 : print ( 'OUTPUTS' ) print ( outputs ) _ , predicted = torch . max ( outputs . data , 1 ) OUTPUTS tensor ([[ - 0.4181 , - 1.0784 , - 0.4840 , - 0.0985 , - 0.2394 , - 0.1801 , - 1.1639 , 2.9352 , - 0.1552 , 0.8852 ], [ 0.5117 , - 0.1099 , 1.5295 , 0.8863 , - 1.8813 , 0.5967 , 1.3632 , - 1.8977 , 0.4183 , - 1.4990 ], [ - 1.0126 , 2.4112 , 0.2373 , 0.0857 , - 0.7007 , - 0.2015 , - 0.3428 , - 0.2548 , 0.1659 , - 0.4703 ], [ 2.8072 , - 2.2973 , - 0.0984 , - 0.4313 , - 0.9619 , 0.8670 , 1.2201 , 0.3752 , - 0.2873 , - 0.3272 ], [ - 0.0343 , - 2.0043 , 0.5081 , - 0.6452 , 1.8647 , - 0.6924 , 0.1435 , 0.4330 , 0.2958 , 1.0339 ], [ - 1.5392 , 2.9070 , 0.2297 , 0.3139 , - 0.6863 , - 0.2734 , - 0.8377 , - 0.1238 , 0.3285 , - 0.3004 ], [ - 1.2037 , - 1.3739 , - 0.5947 , 0.3530 , 1.4205 , 0.0593 , - 0.7307 , 0.6642 , 0.3937 , 0.8004 ], [ - 1.4439 , - 0.3284 , - 0.7652 , - 0.0952 , 0.9323 , 0.3006 , 0.0238 , - 0.0810 , 0.0612 , 1.3295 ], [ 0.5409 , - 0.5266 , 0.9914 , - 1.2369 , 0.6583 , 0.0992 , 0.8525 , - 1.0562 , 0.2013 , 0.0462 ], [ - 0.6548 , - 0.7253 , - 0.9825 , - 1.1663 , 0.9076 , - 0.0694 , - 0.3708 , 1.8270 , 0.2457 , 1.5921 ], [ 3.2147 , - 1.7689 , 0.8531 , 1.2320 , - 0.8126 , 1.1251 , - 0.2776 , - 1.4244 , 0.5930 , - 1.6183 ], [ 0.7470 , - 0.5545 , 1.0251 , 0.0529 , 0.4384 , - 0.5934 , 0.7666 , - 1.0084 , 0.5313 , - 0.3465 ], [ - 0.7916 , - 1.7064 , - 0.7805 , - 1.1588 , 1.3284 , - 0.1708 , - 0.2092 , 0.9495 , 0.1033 , 2.0208 ], [ 3.0602 , - 2.3578 , - 0.2576 , - 0.2198 , - 0.2372 , 0.9765 , - 0.1514 , - 0.5380 , 0.7970 , 0.1374 ], [ - 1.2613 , 2.8594 , - 0.0874 , 0.1974 , - 1.2018 , - 0.0064 , - 0.0923 , - 0.2142 , 0.2575 , - 0.3218 ], [ 0.4348 , - 0.7216 , 0.0021 , 1.2864 , - 0.5062 , 0.7761 , - 0.3236 , - 0.5667 , 0.5431 , - 0.7781 ], [ - 0.2157 , - 2.0200 , 0.1829 , - 0.6882 , 1.3815 , - 0.7609 , - 0.0902 , 0.8647 , 0.3679 , 1.8843 ], [ 0.0950 , - 1.5009 , - 0.6347 , 0.3662 , - 0.4679 , - 0.0359 , - 0.7671 , 2.7155 , - 0.3991 , 0.5737 ], [ - 0.7005 , - 0.5366 , - 0.0434 , 1.1289 , - 0.5873 , 0.2555 , 0.8187 , - 0.6557 , 0.1241 , - 0.4297 ], [ - 1.0635 , - 1.5991 , - 0.4677 , - 0.1231 , 2.0445 , 0.1128 , - 0.1825 , 0.1075 , 0.0348 , 1.4317 ], [ - 1.0319 , - 0.1595 , - 1.3415 , 0.1095 , 0.5339 , 0.1973 , - 1.3272 , 1.5765 , 0.4784 , 1.4176 ], [ - 0.4928 , - 1.5653 , - 0.0672 , 0.3325 , 0.5359 , 0.5368 , 2.1542 , - 1.4276 , 0.3605 , 0.0587 ], [ - 0.4761 , 0.2958 , 0.6597 , - 0.2658 , 1.1279 , - 1.0676 , 1.2506 , - 0.2059 , - 0.1489 , 0.1051 ], [ - 0.0764 , - 0.9274 , - 0.6838 , 0.3464 , - 0.2656 , 1.4099 , 0.4486 , - 0.9527 , 0.5682 , 0.0156 ], [ - 0.6900 , - 0.9611 , 0.1395 , - 0.0079 , 1.5424 , - 0.3208 , - 0.2682 , 0.3586 , - 0.2771 , 1.0389 ], [ 4.3606 , - 2.8621 , 0.6310 , - 0.9657 , - 0.2486 , 1.2009 , 1.1873 , - 0.8255 , - 0.2103 , - 1.2172 ], [ - 0.1000 , - 1.4268 , - 0.4627 , - 0.1041 , 0.2959 , - 0.1392 , - 0.6855 , 1.8622 , - 0.2580 , 1.1347 ], [ - 0.3625 , - 2.1323 , - 0.2224 , - 0.8754 , 2.4684 , 0.0295 , 0.1161 , - 0.2660 , 0.3037 , 1.4570 ], [ 2.8688 , - 2.4517 , 0.1782 , 1.1149 , - 1.0898 , 1.1062 , - 0.0681 , - 0.5697 , 0.8888 , - 0.6965 ], [ - 1.0429 , 1.4446 , - 0.3349 , 0.1254 , - 0.5017 , 0.2286 , 0.2328 , - 0.3290 , 0.3949 , - 0.2586 ], [ - 0.8476 , - 0.0004 , - 1.1003 , 2.2806 , - 1.2226 , 0.9251 , - 0.3165 , 0.4957 , 0.0690 , 0.0232 ], [ - 0.9108 , 1.1355 , - 0.2715 , 0.2233 , - 0.3681 , 0.1442 , - 0.0001 , - 0.0174 , 0.1454 , 0.2286 ], [ - 1.0663 , - 0.8466 , - 0.7147 , 2.5685 , - 0.2090 , 1.2993 , - 0.3057 , - 0.8314 , 0.7046 , - 0.0176 ], [ 1.7013 , - 1.8051 , 0.7541 , - 1.5248 , 0.8972 , 0.1518 , 1.4876 , - 0.8454 , - 0.2022 , - 0.2829 ], [ - 0.8179 , - 0.1239 , 0.8630 , - 0.2137 , - 0.2275 , - 0.5411 , - 1.3448 , 1.7354 , 0.7751 , 0.6234 ], [ 0.6515 , - 1.0431 , 2.7165 , 0.1873 , - 1.0623 , 0.1286 , 0.3597 , - 0.2739 , 0.3871 , - 1.6699 ], [ - 0.2828 , - 1.4663 , 0.1182 , - 0.0896 , - 0.3640 , - 0.5129 , - 0.4905 , 2.2914 , - 0.2227 , 0.9463 ], [ - 1.2596 , 2.0468 , - 0.4405 , - 0.0411 , - 0.8073 , 0.0490 , - 0.0604 , - 0.1206 , 0.3504 , - 0.1059 ], [ 0.6089 , 0.5885 , 0.7898 , 1.1318 , - 1.9008 , 0.5875 , 0.4227 , - 1.1815 , 0.5652 , - 1.3590 ], [ - 1.4551 , 2.9537 , - 0.2805 , 0.2372 , - 1.4180 , 0.0297 , - 0.1515 , - 0.6111 , 0.6140 , - 0.3354 ], [ - 0.7182 , 1.6778 , 0.0553 , 0.0461 , - 0.5446 , - 0.0338 , - 0.0215 , - 0.0881 , 0.1506 , - 0.2107 ], [ - 0.8027 , - 0.7854 , - 0.1275 , - 0.3177 , - 0.1600 , - 0.1964 , - 0.6084 , 2.1285 , - 0.1815 , 1.1911 ], [ - 2.0656 , - 0.4959 , - 0.1154 , - 0.1363 , 2.2426 , - 0.7441 , - 0.8413 , 0.4675 , 0.3269 , 1.7279 ], [ - 0.3004 , 1.0166 , 1.1175 , - 0.0618 , - 0.0937 , - 0.4221 , 0.1943 , - 1.1020 , 0.3670 , - 0.4683 ], [ - 1.0720 , 0.2252 , 0.0175 , 1.3644 , - 0.7409 , 0.4655 , 0.5439 , 0.0380 , 0.1279 , - 0.2302 ], [ 0.2409 , - 1.2622 , - 0.6336 , 1.8240 , - 0.5951 , 1.3408 , 0.2130 , - 1.3789 , 0.8363 , - 0.2101 ], [ - 1.3849 , 0.3773 , - 0.0585 , 0.6896 , - 0.0998 , 0.2804 , 0.0696 , - 0.2529 , 0.3143 , 0.3409 ], [ - 0.9103 , - 0.1578 , 1.6673 , - 0.4817 , 0.4088 , - 0.5484 , 0.6103 , - 0.2287 , - 0.0665 , 0.0055 ], [ - 1.1692 , - 2.8531 , - 1.2499 , - 0.0257 , 2.8580 , 0.2616 , - 0.7122 , - 0.0551 , 0.8112 , 2.3233 ], [ - 0.2790 , - 1.9494 , 0.6096 , - 0.5653 , 2.2792 , - 1.0687 , 0.1634 , 0.3122 , 0.1053 , 1.0884 ], [ 0.1267 , - 1.2297 , - 0.1315 , 0.2428 , - 0.5436 , 0.4123 , 2.3060 , - 0.9278 , - 0.1528 , - 0.4224 ], [ - 0.0235 , - 0.9137 , - 0.1457 , 1.6858 , - 0.7552 , 0.7293 , 0.2510 , - 0.3955 , - 0.2187 , - 0.1505 ], [ 0.5643 , - 1.2783 , - 1.4149 , 0.0304 , 0.8375 , 1.5018 , 0.0338 , - 0.3875 , - 0.0117 , 0.5751 ], [ 0.2926 , - 0.7486 , - 0.3238 , 1.0384 , 0.0308 , 0.6792 , - 0.0170 , - 0.5797 , 0.2819 , - 0.3510 ], [ 0.1219 , - 0.5862 , 1.5817 , - 0.1297 , 0.4730 , - 0.9171 , 0.7886 , - 0.7022 , - 0.0501 , - 0.2812 ], [ 1.7587 , - 2.4511 , - 0.7369 , 0.4082 , - 0.6426 , 1.1784 , 0.6052 , - 0.7178 , 1.6161 , - 0.2220 ], [ - 0.1267 , - 2.6719 , 0.0505 , - 0.4972 , 2.9027 , - 0.1461 , 0.2807 , - 0.2921 , 0.2231 , 1.1327 ], [ - 0.9892 , 2.4401 , 0.1274 , 0.2838 , - 0.7535 , - 0.1684 , - 0.6493 , - 0.1908 , 0.2290 , - 0.2150 ], [ - 0.2071 , - 2.1351 , - 0.9191 , - 0.9309 , 1.7747 , - 0.3046 , 0.0183 , 1.0136 , - 0.1016 , 2.1288 ], [ - 0.0103 , 0.3280 , - 0.6974 , - 0.2504 , 0.3187 , 0.4390 , - 0.1879 , 0.3954 , 0.2332 , - 0.1971 ], [ - 0.2280 , - 1.6754 , - 0.7438 , 0.5078 , 0.2544 , - 0.1020 , - 0.2503 , 2.0799 , - 0.5033 , 0.5890 ], [ 0.3972 , - 0.9369 , 1.2696 , - 1.6713 , - 0.4159 , - 0.0221 , 0.6489 , - 0.4777 , 1.2497 , 0.3931 ], [ - 0.7566 , - 0.8230 , - 0.0785 , - 0.3083 , 0.7821 , 0.1880 , 0.1037 , - 0.0956 , 0.4219 , 1.0798 ], [ - 1.0328 , - 0.1700 , 1.3806 , 0.5445 , - 0.2624 , - 0.0780 , - 0.3595 , - 0.6253 , 0.4309 , 0.1813 ], [ - 1.0360 , - 0.4704 , 0.1948 , - 0.7066 , 0.6600 , - 0.4633 , - 0.3602 , 1.7494 , 0.1522 , 0.6086 ], [ - 1.2032 , - 0.7903 , - 0.5754 , 0.4722 , 0.6068 , 0.5752 , 0.2151 , - 0.2495 , 0.3420 , 0.9278 ], [ 0.2247 , - 0.1361 , 0.9374 , - 0.1543 , 0.4921 , - 0.6553 , 0.5885 , 0.2617 , - 0.2216 , - 0.3736 ], [ - 0.2867 , - 1.4486 , 0.6658 , - 0.8755 , 2.3195 , - 0.7627 , - 0.2132 , 0.2488 , 0.3484 , 1.0860 ], [ - 1.4031 , - 0.4518 , - 0.3181 , 2.8268 , - 0.5371 , 1.0154 , - 0.9247 , - 0.7385 , 1.1031 , 0.0422 ], [ 2.8604 , - 1.5413 , 0.6241 , - 0.8017 , - 1.4104 , 0.6314 , 0.4614 , - 0.0218 , - 0.3411 , - 0.2609 ], [ 0.2113 , - 1.2348 , - 0.8535 , - 0.1041 , - 0.2703 , - 0.1294 , - 0.7057 , 2.7552 , - 0.4429 , 0.4517 ], [ 4.5191 , - 2.7407 , 1.1091 , 0.3975 , - 0.9456 , 1.2277 , 0.3616 , - 1.6564 , 0.5063 , - 1.4274 ], [ 1.4615 , - 1.0765 , 1.8388 , 1.5006 , - 1.2351 , 0.2781 , 0.2830 , - 0.8491 , 0.2222 , - 1.7779 ], [ - 1.2160 , 0.8502 , 0.2413 , - 0.0798 , - 0.7880 , - 0.4286 , - 0.8060 , 0.7194 , 1.2663 , 0.6412 ], [ - 1.3318 , 2.3388 , - 0.4003 , - 0.1094 , - 1.0285 , 0.1021 , - 0.0388 , - 0.0497 , 0.5137 , - 0.2507 ], [ - 1.7853 , 0.5884 , - 0.6108 , - 0.5557 , 0.8696 , - 0.6226 , - 0.7983 , 1.7169 , - 0.0145 , 0.8231 ], [ - 0.1739 , 0.1562 , - 0.2933 , 2.3195 , - 0.9480 , 1.2019 , - 0.4834 , - 1.0567 , 0.5685 , - 0.6841 ], [ - 0.7920 , - 0.3339 , 0.7452 , - 0.6529 , - 0.3307 , - 0.6092 , - 0.0950 , 1.7311 , - 0.3481 , 0.3801 ], [ - 1.7810 , 1.0676 , - 0.7611 , 0.3658 , - 0.0431 , - 0.1012 , - 0.6048 , 0.3089 , 0.9998 , 0.7164 ], [ - 0.5856 , - 0.5261 , - 0.4859 , - 1.0551 , - 0.1838 , - 0.2144 , - 1.2599 , 3.3891 , 0.4691 , 0.7566 ], [ - 0.4984 , - 1.7770 , - 1.1998 , - 0.1075 , 1.0882 , 0.4539 , - 0.5651 , 1.4381 , - 0.5678 , 1.7479 ], [ 0.2938 , - 1.8536 , 0.4259 , - 0.5429 , 0.0066 , 0.4120 , 2.3793 , - 0.3666 , - 0.2604 , 0.0382 ], [ - 0.4080 , - 0.9851 , 4.0264 , 0.1099 , - 0.1766 , - 1.1557 , 0.6419 , - 0.8147 , 0.7535 , - 1.1452 ], [ - 0.4636 , - 1.7323 , - 0.6433 , - 0.0274 , 0.7227 , - 0.1799 , - 0.9336 , 2.1881 , - 0.2073 , 1.6522 ], [ - 0.9617 , - 0.0348 , - 0.3980 , - 0.4738 , 0.7790 , 0.4671 , - 0.6115 , - 0.7067 , 1.3036 , 0.4923 ], [ - 1.0151 , - 2.5385 , - 0.6072 , 0.2902 , 3.1570 , 0.1062 , - 0.2169 , - 0.4491 , 0.6326 , 1.6829 ], [ - 1.8852 , 0.6066 , - 0.2840 , - 0.4475 , - 0.1147 , - 0.7858 , - 1.1805 , 3.0723 , 0.3960 , 0.9720 ], [ 0.0344 , - 1.4878 , - 0.9675 , 1.9649 , - 0.3146 , 1.2183 , 0.6730 , - 0.3650 , 0.0646 , - 0.0898 ], [ - 0.2118 , - 2.0350 , 0.9917 , - 0.8993 , 1.2334 , - 0.6723 , 2.5847 , - 0.0454 , - 0.4149 , 0.3927 ], [ - 1.7365 , 3.0447 , 0.5115 , 0.0786 , - 0.7544 , - 0.2158 , - 0.4876 , - 0.2891 , 0.5089 , - 0.6719 ], [ 0.3652 , - 0.5457 , - 0.1167 , 2.9056 , - 1.1622 , 0.8192 , - 1.3245 , - 0.6414 , 0.8097 , - 0.4958 ], [ - 0.8755 , - 0.6983 , 0.2208 , - 0.6463 , 0.5276 , 0.1145 , 2.7229 , - 1.0316 , 0.1905 , 0.2090 ], [ - 0.9702 , 0.1265 , - 0.0007 , - 0.5106 , 0.4970 , - 0.0804 , 0.0017 , 0.0607 , 0.6164 , 0.4490 ], [ - 0.8271 , - 0.6822 , - 0.7434 , 2.6457 , - 1.6143 , 1.1486 , - 1.0705 , 0.5611 , 0.6422 , 0.1250 ], [ - 1.9979 , 1.8175 , - 0.1658 , - 0.0343 , - 0.6292 , 0.1774 , 0.3150 , - 0.4633 , 0.9266 , 0.0252 ], [ - 0.9039 , - 0.6030 , - 0.2173 , - 1.1768 , 2.3198 , - 0.5072 , 0.3418 , - 0.1551 , 0.1282 , 1.4250 ], [ - 0.9891 , 0.5212 , - 0.4518 , 0.3267 , - 0.0759 , 0.3826 , - 0.0341 , 0.0382 , 0.2451 , 0.3658 ], [ - 2.1217 , 1.5102 , - 0.7828 , 0.3554 , - 0.4192 , - 0.0772 , 0.0578 , 0.8070 , 0.1701 , 0.5880 ], [ 1.0665 , - 1.3826 , 0.6243 , - 0.8096 , - 0.4227 , 0.5925 , 1.8112 , - 0.9946 , 0.2010 , - 0.7731 ], [ - 1.1263 , - 1.7484 , 0.0041 , - 0.5439 , 1.7242 , - 0.9475 , - 0.3835 , 0.8452 , 0.3077 , 2.2689 ]]) Printing output size This produces a 100x10 matrix because each iteration has a batch size of 100 and each prediction across the 10 classes, with the largest number indicating the likely number it is predicting. iter_test = 0 for images , labels in test_loader : iter_test += 1 images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () outputs = model ( images ) if iter_test == 1 : print ( 'OUTPUTS' ) print ( outputs . size ()) _ , predicted = torch . max ( outputs . data , 1 ) OUTPUTS torch . Size ([ 100 , 10 ]) Printing one output This would be a 1x10 matrix where the largest number is what the model thinks the image is. Here we can see that in the tensor, position 7 has the largest number, indicating the model thinks the image is 7. number 0: -0.4181 number 1: -1.0784 ... number 7: 2.9352 iter_test = 0 for images , labels in test_loader : iter_test += 1 images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () outputs = model ( images ) if iter_test == 1 : print ( 'OUTPUTS' ) print ( outputs [ 0 , :]) _ , predicted = torch . max ( outputs . data , 1 ) OUTPUTS tensor([-0.4181, -1.0784, -0.4840, -0.0985, -0.2394, -0.1801, -1.1639, 2.9352, -0.1552, 0.8852]) Printing prediction output Because our output is of size 100 (our batch size), our prediction size would also of the size 100. iter_test = 0 for images , labels in test_loader : iter_test += 1 images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () outputs = model ( images ) _ , predicted = torch . max ( outputs . data , 1 ) if iter_test == 1 : print ( 'PREDICTION' ) print ( predicted . size ()) PREDICTION torch . Size ([ 100 ]) Print prediction value We are printing our prediction which as verified above, should be digit 7. iter_test = 0 for images , labels in test_loader : iter_test += 1 images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () outputs = model ( images ) _ , predicted = torch . max ( outputs . data , 1 ) if iter_test == 1 : print ( 'PREDICTION' ) print ( predicted [ 0 ]) PREDICTION tensor ( 7 ) Print prediction, label and label size We are trying to show what we are predicting and the actual values. In this case, we're predicting the right value 7! iter_test = 0 for images , labels in test_loader : iter_test += 1 images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () outputs = model ( images ) _ , predicted = torch . max ( outputs . data , 1 ) if iter_test == 1 : print ( 'PREDICTION' ) print ( predicted [ 0 ]) print ( 'LABEL SIZE' ) print ( labels . size ()) print ( 'LABEL FOR IMAGE 0' ) print ( labels [ 0 ]) PREDICTION tensor ( 7 ) LABEL SIZE torch . Size ([ 100 ]) LABEL FOR IMAGE 0 tensor ( 7 ) Print second prediction and ground truth Again, the prediction is correct. Naturally, as our model is quite competent in this simple task. iter_test = 0 for images , labels in test_loader : iter_test += 1 images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () outputs = model ( images ) _ , predicted = torch . max ( outputs . data , 1 ) if iter_test == 1 : print ( 'PREDICTION' ) print ( predicted [ 1 ]) print ( 'LABEL SIZE' ) print ( labels . size ()) print ( 'LABEL FOR IMAGE 1' ) print ( labels [ 1 ]) PREDICTION tensor ( 2 ) LABEL SIZE torch . Size ([ 100 ]) LABEL FOR IMAGE 1 tensor ( 2 ) Print accuracy Now we know what each object represents, we can understand how we arrived at our accuracy numbers. One last thing to note is that correct.item() has this syntax is because correct is a PyTorch tensor and to get the value to compute with total which is an integer, we need to do this. correct = 0 total = 0 iter_test = 0 for images , labels in test_loader : iter_test += 1 images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () outputs = model ( images ) _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * ( correct . item () / total ) print ( accuracy ) 82.94 Explanation of Python's .sum() function Python's .sum() function allows you to do a comparison between two matrices and sum the ones that return True or in our case, those predictions that match actual labels (correct predictions). # Explaining .sum() python built-in function # correct += (predicted == labels).sum() import numpy as np a = np . ones (( 10 )) print ( a ) b = np . ones (( 10 )) print ( b ) print ( a == b ) print (( a == b ) . sum ()) # matrix a [ 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. ] # matrix b [ 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. ] # boolean array [ True True True True True True True True True True ] # number of elementswhere a matches b 10 Saving Model \u00b6 Saving PyTorch model This is how you save your model. Feel free to just change save_model = True to save your model save_model = False if save_model is True : # Saves only parameters torch . save ( model . state_dict (), 'awesome_model.pkl' ) Building a Logistic Regression Model with PyTorch (GPU) \u00b6 CPU version The usual 7-step process, getting repetitive by now which we like. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class LogisticRegressionModel ( nn . Module ): def __init__ ( self , input_size , num_classes ): super ( LogisticRegressionModel , self ) . __init__ () self . linear = nn . Linear ( input_dim , output_dim ) def forward ( self , x ): out = self . linear ( x ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 output_dim = 10 model = LogisticRegressionModel ( input_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.001 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () labels = labels # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits # 100 x 10 outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value # 100 x 1 _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct . item () / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 1.876196026802063 . Accuracy : 64.44 Iteration : 1000. Loss : 1.5153584480285645 . Accuracy : 75.68 Iteration : 1500. Loss : 1.3521136045455933 . Accuracy : 78.98 Iteration : 2000. Loss : 1.2136967182159424 . Accuracy : 80.95 Iteration : 2500. Loss : 1.0934826135635376 . Accuracy : 81.97 Iteration : 3000. Loss : 1.024120569229126 . Accuracy : 82.49 GPU version 2 things must be on GPU - model - tensors Remember step 4 and 7 will be affected and this will be the same for all model building moving forward. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class LogisticRegressionModel ( nn . Module ): def __init__ ( self , input_size , num_classes ): super ( LogisticRegressionModel , self ) . __init__ () self . linear = nn . Linear ( input_dim , output_dim ) def forward ( self , x ): out = self . linear ( x ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 output_dim = 10 model = LogisticRegressionModel ( input_dim , output_dim ) ####################### # USE GPU FOR MODEL # ####################### device = torch . device ( \"cuda:0\" if torch . cuda . is_available () else \"cpu\" ) model . to ( device ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.001 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): ####################### # USE GPU FOR MODEL # ####################### images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () . to ( device ) labels = labels . to ( device ) # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : ####################### # USE GPU FOR MODEL # ####################### images = images . view ( - 1 , 28 * 28 ) . to ( device ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) ####################### # USE GPU FOR MODEL # ####################### # Total correct predictions if torch . cuda . is_available (): correct += ( predicted . cpu () == labels . cpu ()) . sum () else : correct += ( predicted == labels ) . sum () accuracy = 100 * correct . item () / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 1.8571407794952393 . Accuracy : 68.99 Iteration : 1000. Loss : 1.5415704250335693 . Accuracy : 75.86 Iteration : 1500. Loss : 1.2755383253097534 . Accuracy : 78.92 Iteration : 2000. Loss : 1.2468739748001099 . Accuracy : 80.72 Iteration : 2500. Loss : 1.0708973407745361 . Accuracy : 81.73 Iteration : 3000. Loss : 1.0359245538711548 . Accuracy : 82.74 Summary \u00b6 We've learnt to... Success Logistic regression basics Problems of linear regression In-depth Logistic Regression Get logits Get softmax Get cross-entropy loss Aim : reduce cross-entropy loss Built a logistic regression model in CPU and GPU Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model Important things to be on GPU model tensors with gradients Citation \u00b6 If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Logistic Regression"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#logistic-regression-with-pytorch","text":"Run Jupyter Notebook You can run the code for this section in this jupyter notebook link .","title":"Logistic Regression with PyTorch"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#about-logistic-regression","text":"","title":"About Logistic Regression"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#logistic-regression-basics","text":"","title":"Logistic Regression Basics"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#classification-algorithm","text":"Example: Spam vs No Spam Input: Bunch of words Output: Probability spam or not","title":"Classification algorithm"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#basic-comparison","text":"Linear regression Output: numeric value given inputs Logistic regression : Output: probability [0, 1] given input belonging to a class","title":"Basic Comparison"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#inputoutput-comparison","text":"Linear regression: Multiplication Input: [1] Output: 2 Input: [2] Output: 4 Trying to model the relationship y = 2x Logistic regression: Spam Input: \"Sign up to get 1 million dollars by tonight\" Output: p = 0.8 Input: \"This is a receipt for your recent purchase with Amazon\" Output: p = 0.3 p: probability it is spam","title":"Input/Output Comparison"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#problems-of-linear-regression","text":"Example Fever Input : temperature Output : fever or no fever Remember Linear regression : minimize error between points and line Linear Regression Problem 1: Fever value can go negative (below 0) and positive (above 1) If you simply tried to do a simple linear regression on this fever problem, you would realize an apparent error. Fever can go beyond 1 and below 0 which does not make sense in this context. import numpy as np import matplotlib.pyplot as plt % matplotlib inline x = [ 1 , 5 , 10 , 10 , 25 , 50 , 70 , 75 , 100 ,] y = [ 0 , 0 , 0 , 0 , 0 , 1 , 1 , 1 , 1 ] colors = np . random . rand ( len ( x )) plt . plot ( np . unique ( x ), np . poly1d ( np . polyfit ( x , y , 1 ))( np . unique ( x ))) plt . ylabel ( \"Fever\" ) plt . xlabel ( \"Temperature\" ) plt . scatter ( x , y , c = colors , alpha = 0.5 ) plt . show () Linear Regression Problem 2: Fever points are not predicted with the presence of outliers Previously at least some points could be properly predicted. However, with the presence of outliers, everything goes wonky for simple linear regression, having no predictive capacity at all. import numpy as np import matplotlib.pyplot as plt x = [ 1 , 5 , 10 , 10 , 25 , 50 , 70 , 75 , 300 ] y = [ 0 , 0 , 0 , 0 , 0 , 1 , 1 , 1 , 1 ] colors = np . random . rand ( len ( x )) plt . plot ( np . unique ( x ), np . poly1d ( np . polyfit ( x , y , 1 ))( np . unique ( x ))) plt . ylabel ( \"Fever\" ) plt . xlabel ( \"Temperature\" ) plt . scatter ( x , y , c = colors , alpha = 0.5 ) plt . show ()","title":"Problems of Linear Regression"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#logistic-regression-in-depth","text":"","title":"Logistic Regression In-Depth"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#predicting-probability","text":"Linear regression doesn't work Instead of predicting direct values: predict probability","title":"Predicting Probability"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#logistic-function-g","text":"\"Two-class logistic regression\" \\(\\boldsymbol{y} = A\\boldsymbol{x} + \\boldsymbol{b}\\) Where \\(\\boldsymbol{y}\\) is a vector comprising the 2-class prediction \\(y_0\\) and \\(y_1\\) Where the labels are \\(y_0 = 0\\) and \\(y_1 = 1\\) Also, it's bolded because it's a vector, not a matrix. \\(g(y_1) = \\frac {1} {1 + e^{-y_1}}\\) \\(g(y_1)\\) = Estimated probability that \\(y = 1\\) \\(g(y_0) = 1 - g(y_1)\\) \\(g(y_0)\\) = Estimated probability that \\(y = 0\\) For our illustration above, we have 4 classes, so we have to use softmax function explained below","title":"Logistic Function g()"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#softmax-function-g","text":"\"Multi-class logistic regression\" Generalization of logistic function, where you can derive back to the logistic function if you've a 2 class classification problem Here, we will use a 4 class example (K = 4) as shown above to be very clear in how it relates back to that simple examaple. \\(\\boldsymbol{y} = A\\boldsymbol{x} + \\boldsymbol{b}\\) Where \\(\\boldsymbol{y}\\) is a vector comprising the 4-class prediction \\(y_0, y_1, y_2, y_3\\) Where the 4 labels (K = 4) are \\(y_0 = 0, y_1 = 1, y_2 = 2, y_3 = 3\\) \\(g(y_i) = \\frac {e^{y_i} } {\\sum^K_i e^{y_i}}\\) where K = 4 because we have 4 classes To put numbers to this equation in relation to the illustration above where we've \\(y_0 = 1.3, y_1 = 1.2, y = 4.5, y = 4.8\\) \\(g(y_0) = \\frac {e^{1.3}}{e^{1.3} + e^{1.2} + e^{4.5} + e^{4.8}} = 0.017\\) \\(g(y_1) = \\frac {e^{1.2}}{e^{1.3} + e^{1.2} + e^{4.5} + e^{4.8}} = 0.015\\) \\(g(y_2) = \\frac {e^{4.5}}{e^{1.3} + e^{1.2} + e^{4.5} + e^{4.8}} = 0.412\\) \\(g(y_3) = \\frac {e^{4.8}}{e^{1.3} + e^{1.2} + e^{4.5} + e^{4.8}} = 0.556\\) \\(g(y_0) + g(y_1) + g(y_2) + g(y_3) = 1.0\\) All softmax outputs have to sum to one as they represent a probability distribution over K classes. Take note how these numbers are not exactly as in the illustration in the softmax box but the concept is important (intentionally made so). \\(y_0\\) and \\(y_1\\) are approximately similar in values and they return similar probabilities. Similarly, \\(y_2\\) and \\(y_3\\) are approximately similar in values and they return similar probabilities. Softmax versus Soft(arg)max Do you know many researchers and anyone in deep learning in general use the term softmax when it should be soft(arg)max. This is because soft(arg)max returns the probability distribution over K classes, a vector. However, softmax only returns the max! This means you will be getting a scalar value versus a probability distribution. According to my friend, Alfredo Canziani (postdoc in NYU under Yann Lecun), it was actually a mistake made in the original paper previously but it was too late because the term softmax was adopted. Full credits to him for this tip.","title":"Softmax Function g()"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#cross-entropy-function-d-for-2-class","text":"Take note that here, \\(S\\) is our softmax outputs and \\(L\\) are our labels \\(D(S, L) = -(L log S + (1-L)log(1-S))\\) If L = 0 (label) \\(D(S, 0) = - log(1-S)\\) \\(- log(1-S)\\) : less positive if \\(S \\longrightarrow 0\\) \\(- log(1-S)\\) : more positive if \\(S \\longrightarrow 1\\) (BIGGER LOSS) If L = 1 (label) \\(D(S, 1) = - log S\\) \\(-log(S)\\) : less positive if \\(S \\longrightarrow 1\\) \\(-log(S)\\) : more positive if \\(S \\longrightarrow 0\\) (BIGGER LOSS) Numerical example of bigger or small loss You get a small error of 1e-5 if your label = 0 and your S is closer to 0 (very correct prediction). import math print ( - math . log ( 1 - 0.00001 )) You get a large error of 11.51 if your label is 0 and S is near to 1 (very wrong prediction). print ( - math . log ( 1 - 0.99999 )) You get a small error of -1e-5 if your label is 1 and S is near 1 (very correct prediction). print ( - math . log ( 0.99999 )) You get a big error of -11.51 if your label is 1 and S is near 0 (very wrong prediction). print ( - math . log ( 0.00001 )) 1.0000050000287824e-05 11.51292546497478 1.0000050000287824e-05 11.512925464970229","title":"Cross Entropy Function D() for 2 Class"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#cross-entropy-function-d-for-more-than-2-class","text":"For the case where we have more than 2 class, we need a more generalized function \\(D(S, L) = - \\sum^K_1 L_i log(S_i)\\) \\(K\\) : number of classes \\(L_i\\) : label of i-th class, 1 if that's the class else 0 \\(S_i\\) : output of softmax for i-th class","title":"Cross Entropy Function D() for More Than 2 Class"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#cross-entropy-loss-over-n-samples","text":"Goal: Minimizing Cross Entropy Loss, L \\(Loss = \\frac {1}{N} \\sum_j^N D_j\\) \\(D_j\\) : j-th sample of cross entropy function \\(D(S, L)\\) \\(N\\) : number of samples \\(Loss\\) : average cross entropy loss over N samples","title":"Cross Entropy Loss over N samples"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#building-a-logistic-regression-model-with-pytorch","text":"","title":"Building a Logistic Regression Model with PyTorch"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#steps","text":"Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model","title":"Steps"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#step-1a-loading-mnist-train-dataset","text":"Images from 1 to 9 Inspect length of training dataset You can easily load MNIST dataset with PyTorch. Here we inspect the training set, where our algorithms will learn from, and you will discover it is made up of 60,000 images. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) len ( train_dataset ) 60000 Inspecting a single image So this is how a single image is represented in numbers. It's actually a 28 pixel x 28 pixel image which is why you would end up with this 28x28 matrix of numbers. train_dataset [ 0 ] ( tensor ([[[ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0118 , 0.0706 , 0.0706 , 0.0706 , 0.4941 , 0.5333 , 0.6863 , 0.1020 , 0.6510 , 1.0000 , 0.9686 , 0.4980 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.1176 , 0.1412 , 0.3686 , 0.6039 , 0.6667 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.8824 , 0.6745 , 0.9922 , 0.9490 , 0.7647 , 0.2510 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.1922 , 0.9333 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.9843 , 0.3647 , 0.3216 , 0.3216 , 0.2196 , 0.1529 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0706 , 0.8588 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.7765 , 0.7137 , 0.9686 , 0.9451 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.3137 , 0.6118 , 0.4196 , 0.9922 , 0.9922 , 0.8039 , 0.0431 , 0.0000 , 0.1686 , 0.6039 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0549 , 0.0039 , 0.6039 , 0.9922 , 0.3529 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.5451 , 0.9922 , 0.7451 , 0.0078 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0431 , 0.7451 , 0.9922 , 0.2745 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.1373 , 0.9451 , 0.8824 , 0.6275 , 0.4235 , 0.0039 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.3176 , 0.9412 , 0.9922 , 0.9922 , 0.4667 , 0.0980 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.1765 , 0.7294 , 0.9922 , 0.9922 , 0.5882 , 0.1059 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0627 , 0.3647 , 0.9882 , 0.9922 , 0.7333 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.9765 , 0.9922 , 0.9765 , 0.2510 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.1804 , 0.5098 , 0.7176 , 0.9922 , 0.9922 , 0.8118 , 0.0078 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.1529 , 0.5804 , 0.8980 , 0.9922 , 0.9922 , 0.9922 , 0.9804 , 0.7137 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0941 , 0.4471 , 0.8667 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.7882 , 0.3059 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0902 , 0.2588 , 0.8353 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.7765 , 0.3176 , 0.0078 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0706 , 0.6706 , 0.8588 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.7647 , 0.3137 , 0.0353 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.2157 , 0.6745 , 0.8863 , 0.9922 , 0.9922 , 0.9922 , 0.9922 , 0.9569 , 0.5216 , 0.0431 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.5333 , 0.9922 , 0.9922 , 0.9922 , 0.8314 , 0.5294 , 0.5176 , 0.0627 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ], [ 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 , 0.0000 ]]]), tensor ( 5 )) Inspecting a single data point in the training dataset When you load MNIST dataset, each data point is actually a tuple containing the image matrix and the label. type ( train_dataset [ 0 ]) tuple Inspecting training dataset first element of tuple This means to access the image, you need to access the first element in the tuple. # Input Matrix train_dataset [ 0 ][ 0 ] . size () # A 28x28 sized image of a digit torch . Size ([ 1 , 28 , 28 ]) Inspecting training dataset second element of tuple The second element actually represents the image's label. Meaning if the second element says 5, it means the 28x28 matrix of numbers represent a digit 5. # Label train_dataset [ 0 ][ 1 ] tensor ( 5 )","title":"Step 1a: Loading MNIST Train Dataset"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#displaying-mnist","text":"Verifying shape of MNIST image As mentioned, a single MNIST image is of the shape 28 pixel x 28 pixel. import matplotlib.pyplot as plt % matplotlib inline import numpy as np train_dataset [ 0 ][ 0 ] . numpy () . shape ( 1 , 28 , 28 ) Plot image of MNIST image show_img = train_dataset [ 0 ][ 0 ] . numpy () . reshape ( 28 , 28 ) plt . imshow ( show_img , cmap = 'gray' ) Second element of tuple shows label As you would expect, the label is 5. # Label train_dataset [ 0 ][ 1 ] tensor ( 5 ) Plot second image of MNIST image show_img = train_dataset [ 1 ][ 0 ] . numpy () . reshape ( 28 , 28 ) plt . imshow ( show_img , cmap = 'gray' ) Second element of tuple shows label We should see 0 here as the label. # Label train_dataset [ 1 ][ 1 ] tensor ( 0 )","title":"Displaying MNIST"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#step-1b-loading-mnist-test-dataset","text":"Show our algorithm works beyond the data we have trained on. Out-of-sample Load test dataset Compared to the 60k images in the training set, the testing set where the model will not be trained on has 10k images to check for its out-of-sample performance. test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) len ( test_dataset ) 10000 Test dataset elements Exactly like the training set, the testing set has 10k tuples containing the 28x28 matrices and their respective labels. type ( test_dataset [ 0 ]) tuple Test dataset first element in tuple This contains the image matrix, similar to the training set. # Image matrix test_dataset [ 0 ][ 0 ] . size () torch . Size ([ 1 , 28 , 28 ]) Plot image sample from test dataset show_img = test_dataset [ 0 ][ 0 ] . numpy () . reshape ( 28 , 28 ) plt . imshow ( show_img , cmap = 'gray' ) Test dataset second element in tuple # Label test_dataset [ 0 ][ 1 ] tensor ( 7 )","title":"Step 1b: Loading MNIST Test Dataset"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#step-2-make-dataset-iterable","text":"Aim: make the dataset iterable totaldata : 60000 minibatch : 100 Number of examples in 1 iteration iterations : 3000 1 iteration: one mini-batch forward & backward pass epochs 1 epoch: running through the whole dataset once \\(epochs = iterations \\div \\frac{totaldata}{minibatch} = 3000 \\div \\frac{60000}{100} = 5\\) Recap training dataset Remember training dataset has 60k images and testing dataset has 10k images. len ( train_dataset ) 60000 Defining epochs When the model goes through the whole 60k images once, learning how to classify 0-9, it's consider 1 epoch. However, there's a concept of batch size where it means the model would look at 100 images before updating the model's weights, thereby learning. When the model updates its weights (parameters) after looking at all the images, this is considered 1 iteration. batch_size = 100 We arbitrarily set 3000 iterations here which means the model would update 3000 times. n_iters = 3000 One epoch consists of 60,000 / 100 = 600 iterations. Because we would like to go through 3000 iterations, this implies we would have 3000 / 600 = 5 epochs as each epoch has 600 iterations. num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) num_epochs 5 Create Iterable Object: Training Dataset train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) Check Iterability import collections isinstance ( train_loader , collections . Iterable ) True Create Iterable Object: Testing Dataset # Iterable object test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) Check iterability of testing dataset isinstance ( test_loader , collections . Iterable ) True Iterate through dataset This is just a simplified example of what we're doing above where we're creating an iterable object lst to loop through so we can access all the images img_1 and img_2 . Above, the equivalent of lst is train_loader and test_loader . img_1 = np . ones (( 28 , 28 )) img_2 = np . ones (( 28 , 28 )) lst = [ img_1 , img_2 ] # Need to iterate # Think of numbers as the images for i in lst : print ( i . shape ) ( 28 , 28 ) ( 28 , 28 )","title":"Step 2: Make Dataset Iterable"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#step-3-building-model","text":"Create model class # Same as linear regression! class LogisticRegressionModel ( nn . Module ): def __init__ ( self , input_dim , output_dim ): super ( LogisticRegressionModel , self ) . __init__ () self . linear = nn . Linear ( input_dim , output_dim ) def forward ( self , x ): out = self . linear ( x ) return out","title":"Step 3: Building Model"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#step-4-instantiate-model-class","text":"Input dimension: Size of image \\(28 \\times 28 = 784\\) Output dimension: 10 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 Check size of dataset This should be 28x28. # Size of images train_dataset [ 0 ][ 0 ] . size () torch . Size ([ 1 , 28 , 28 ]) Instantiate model class based on input and out dimensions As we're trying to classify digits 0-9 a total of 10 classes, our output dimension is 10. And we're feeding the model with 28x28 images, hence our input dimension is 28x28. input_dim = 28 * 28 output_dim = 10 model = LogisticRegressionModel ( input_dim , output_dim )","title":"Step 4: Instantiate Model Class"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#step-5-instantiate-loss-class","text":"Logistic Regression : Cross Entropy Loss Linear Regression: MSE Create Cross Entry Loss Class Unlike linear regression, we do not use MSE here, we need Cross Entry Loss to calculate our loss before we backpropagate and update our parameters. criterion = nn . CrossEntropyLoss () What happens in nn.CrossEntropyLoss()? It does 2 things at the same time. 1. Computes softmax (logistic/softmax function) 2. Computes cross entropy","title":"Step 5: Instantiate Loss Class"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#step-6-instantiate-optimizer-class","text":"Simplified equation \\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\) \\(\\theta\\) : parameters (our variables) \\(\\eta\\) : learning rate (how fast we want to learn) \\(\\nabla_\\theta\\) : parameters' gradients Even simplier equation parameters = parameters - learning_rate * parameters_gradients At every iteration, we update our model's parameters Create optimizer Similar to what we've covered above, this calculates the parameters' gradients and update them subsequently. learning_rate = 0.001 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) Parameters In-Depth You'll realize we have 2 sets of parameters, 10x784 which is A and 10x1 which is b in the \\(y = AX + b\\) equation where X is our input of size 784. We'll go into details subsequently how these parameters interact with our input to produce our 10x1 output. # Type of parameter object print ( model . parameters ()) # Length of parameters print ( len ( list ( model . parameters ()))) # FC 1 Parameters print ( list ( model . parameters ())[ 0 ] . size ()) # FC 1 Bias Parameters print ( list ( model . parameters ())[ 1 ] . size ()) < generator object Module . parameters at 0x7ff7c884f830 > 2 torch . Size ([ 10 , 784 ]) torch . Size ([ 10 ]) Quick Matrix Product Review Example 1: matrix product \\(A: (100, 10)\\) \\(B: (10, 1)\\) \\(A \\cdot B = (100, 10) \\cdot (10, 1) = (100, 1)\\) Example 2: matrix product \\(A: (50, 5)\\) \\(B: (5, 2)\\) \\(A \\cdot B = (50, 5) \\cdot (5, 2) = (50, 2)\\) Example 3: element-wise addition \\(A: (10, 1)\\) \\(B: (10, 1)\\) \\(A + B = (10, 1)\\)","title":"Step 6: Instantiate Optimizer Class"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#step-7-train-model","text":"7 step process for training models Process Convert inputs/labels to tensors with gradients Clear gradient buffets Get output given inputs Get loss Get gradients w.r.t. parameters Update parameters using gradients parameters = parameters - learning_rate * parameters_gradients REPEAT iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () labels = labels # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 1.8513233661651611 . Accuracy : 70 Iteration : 1000. Loss : 1.5732524394989014 . Accuracy : 77 Iteration : 1500. Loss : 1.3840199708938599 . Accuracy : 79 Iteration : 2000. Loss : 1.1711134910583496 . Accuracy : 81 Iteration : 2500. Loss : 1.1094708442687988 . Accuracy : 82 Iteration : 3000. Loss : 1.002761721611023 . Accuracy : 82","title":"Step 7: Train Model"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#break-down-accuracy-calculation","text":"Printing outputs of our model As we've trained our model, we can extract the accuracy calculation portion to understand what's happening without re-training the model. This would print out the output of the model's predictions on your notebook. iter_test = 0 for images , labels in test_loader : iter_test += 1 images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () outputs = model ( images ) if iter_test == 1 : print ( 'OUTPUTS' ) print ( outputs ) _ , predicted = torch . max ( outputs . data , 1 ) OUTPUTS tensor ([[ - 0.4181 , - 1.0784 , - 0.4840 , - 0.0985 , - 0.2394 , - 0.1801 , - 1.1639 , 2.9352 , - 0.1552 , 0.8852 ], [ 0.5117 , - 0.1099 , 1.5295 , 0.8863 , - 1.8813 , 0.5967 , 1.3632 , - 1.8977 , 0.4183 , - 1.4990 ], [ - 1.0126 , 2.4112 , 0.2373 , 0.0857 , - 0.7007 , - 0.2015 , - 0.3428 , - 0.2548 , 0.1659 , - 0.4703 ], [ 2.8072 , - 2.2973 , - 0.0984 , - 0.4313 , - 0.9619 , 0.8670 , 1.2201 , 0.3752 , - 0.2873 , - 0.3272 ], [ - 0.0343 , - 2.0043 , 0.5081 , - 0.6452 , 1.8647 , - 0.6924 , 0.1435 , 0.4330 , 0.2958 , 1.0339 ], [ - 1.5392 , 2.9070 , 0.2297 , 0.3139 , - 0.6863 , - 0.2734 , - 0.8377 , - 0.1238 , 0.3285 , - 0.3004 ], [ - 1.2037 , - 1.3739 , - 0.5947 , 0.3530 , 1.4205 , 0.0593 , - 0.7307 , 0.6642 , 0.3937 , 0.8004 ], [ - 1.4439 , - 0.3284 , - 0.7652 , - 0.0952 , 0.9323 , 0.3006 , 0.0238 , - 0.0810 , 0.0612 , 1.3295 ], [ 0.5409 , - 0.5266 , 0.9914 , - 1.2369 , 0.6583 , 0.0992 , 0.8525 , - 1.0562 , 0.2013 , 0.0462 ], [ - 0.6548 , - 0.7253 , - 0.9825 , - 1.1663 , 0.9076 , - 0.0694 , - 0.3708 , 1.8270 , 0.2457 , 1.5921 ], [ 3.2147 , - 1.7689 , 0.8531 , 1.2320 , - 0.8126 , 1.1251 , - 0.2776 , - 1.4244 , 0.5930 , - 1.6183 ], [ 0.7470 , - 0.5545 , 1.0251 , 0.0529 , 0.4384 , - 0.5934 , 0.7666 , - 1.0084 , 0.5313 , - 0.3465 ], [ - 0.7916 , - 1.7064 , - 0.7805 , - 1.1588 , 1.3284 , - 0.1708 , - 0.2092 , 0.9495 , 0.1033 , 2.0208 ], [ 3.0602 , - 2.3578 , - 0.2576 , - 0.2198 , - 0.2372 , 0.9765 , - 0.1514 , - 0.5380 , 0.7970 , 0.1374 ], [ - 1.2613 , 2.8594 , - 0.0874 , 0.1974 , - 1.2018 , - 0.0064 , - 0.0923 , - 0.2142 , 0.2575 , - 0.3218 ], [ 0.4348 , - 0.7216 , 0.0021 , 1.2864 , - 0.5062 , 0.7761 , - 0.3236 , - 0.5667 , 0.5431 , - 0.7781 ], [ - 0.2157 , - 2.0200 , 0.1829 , - 0.6882 , 1.3815 , - 0.7609 , - 0.0902 , 0.8647 , 0.3679 , 1.8843 ], [ 0.0950 , - 1.5009 , - 0.6347 , 0.3662 , - 0.4679 , - 0.0359 , - 0.7671 , 2.7155 , - 0.3991 , 0.5737 ], [ - 0.7005 , - 0.5366 , - 0.0434 , 1.1289 , - 0.5873 , 0.2555 , 0.8187 , - 0.6557 , 0.1241 , - 0.4297 ], [ - 1.0635 , - 1.5991 , - 0.4677 , - 0.1231 , 2.0445 , 0.1128 , - 0.1825 , 0.1075 , 0.0348 , 1.4317 ], [ - 1.0319 , - 0.1595 , - 1.3415 , 0.1095 , 0.5339 , 0.1973 , - 1.3272 , 1.5765 , 0.4784 , 1.4176 ], [ - 0.4928 , - 1.5653 , - 0.0672 , 0.3325 , 0.5359 , 0.5368 , 2.1542 , - 1.4276 , 0.3605 , 0.0587 ], [ - 0.4761 , 0.2958 , 0.6597 , - 0.2658 , 1.1279 , - 1.0676 , 1.2506 , - 0.2059 , - 0.1489 , 0.1051 ], [ - 0.0764 , - 0.9274 , - 0.6838 , 0.3464 , - 0.2656 , 1.4099 , 0.4486 , - 0.9527 , 0.5682 , 0.0156 ], [ - 0.6900 , - 0.9611 , 0.1395 , - 0.0079 , 1.5424 , - 0.3208 , - 0.2682 , 0.3586 , - 0.2771 , 1.0389 ], [ 4.3606 , - 2.8621 , 0.6310 , - 0.9657 , - 0.2486 , 1.2009 , 1.1873 , - 0.8255 , - 0.2103 , - 1.2172 ], [ - 0.1000 , - 1.4268 , - 0.4627 , - 0.1041 , 0.2959 , - 0.1392 , - 0.6855 , 1.8622 , - 0.2580 , 1.1347 ], [ - 0.3625 , - 2.1323 , - 0.2224 , - 0.8754 , 2.4684 , 0.0295 , 0.1161 , - 0.2660 , 0.3037 , 1.4570 ], [ 2.8688 , - 2.4517 , 0.1782 , 1.1149 , - 1.0898 , 1.1062 , - 0.0681 , - 0.5697 , 0.8888 , - 0.6965 ], [ - 1.0429 , 1.4446 , - 0.3349 , 0.1254 , - 0.5017 , 0.2286 , 0.2328 , - 0.3290 , 0.3949 , - 0.2586 ], [ - 0.8476 , - 0.0004 , - 1.1003 , 2.2806 , - 1.2226 , 0.9251 , - 0.3165 , 0.4957 , 0.0690 , 0.0232 ], [ - 0.9108 , 1.1355 , - 0.2715 , 0.2233 , - 0.3681 , 0.1442 , - 0.0001 , - 0.0174 , 0.1454 , 0.2286 ], [ - 1.0663 , - 0.8466 , - 0.7147 , 2.5685 , - 0.2090 , 1.2993 , - 0.3057 , - 0.8314 , 0.7046 , - 0.0176 ], [ 1.7013 , - 1.8051 , 0.7541 , - 1.5248 , 0.8972 , 0.1518 , 1.4876 , - 0.8454 , - 0.2022 , - 0.2829 ], [ - 0.8179 , - 0.1239 , 0.8630 , - 0.2137 , - 0.2275 , - 0.5411 , - 1.3448 , 1.7354 , 0.7751 , 0.6234 ], [ 0.6515 , - 1.0431 , 2.7165 , 0.1873 , - 1.0623 , 0.1286 , 0.3597 , - 0.2739 , 0.3871 , - 1.6699 ], [ - 0.2828 , - 1.4663 , 0.1182 , - 0.0896 , - 0.3640 , - 0.5129 , - 0.4905 , 2.2914 , - 0.2227 , 0.9463 ], [ - 1.2596 , 2.0468 , - 0.4405 , - 0.0411 , - 0.8073 , 0.0490 , - 0.0604 , - 0.1206 , 0.3504 , - 0.1059 ], [ 0.6089 , 0.5885 , 0.7898 , 1.1318 , - 1.9008 , 0.5875 , 0.4227 , - 1.1815 , 0.5652 , - 1.3590 ], [ - 1.4551 , 2.9537 , - 0.2805 , 0.2372 , - 1.4180 , 0.0297 , - 0.1515 , - 0.6111 , 0.6140 , - 0.3354 ], [ - 0.7182 , 1.6778 , 0.0553 , 0.0461 , - 0.5446 , - 0.0338 , - 0.0215 , - 0.0881 , 0.1506 , - 0.2107 ], [ - 0.8027 , - 0.7854 , - 0.1275 , - 0.3177 , - 0.1600 , - 0.1964 , - 0.6084 , 2.1285 , - 0.1815 , 1.1911 ], [ - 2.0656 , - 0.4959 , - 0.1154 , - 0.1363 , 2.2426 , - 0.7441 , - 0.8413 , 0.4675 , 0.3269 , 1.7279 ], [ - 0.3004 , 1.0166 , 1.1175 , - 0.0618 , - 0.0937 , - 0.4221 , 0.1943 , - 1.1020 , 0.3670 , - 0.4683 ], [ - 1.0720 , 0.2252 , 0.0175 , 1.3644 , - 0.7409 , 0.4655 , 0.5439 , 0.0380 , 0.1279 , - 0.2302 ], [ 0.2409 , - 1.2622 , - 0.6336 , 1.8240 , - 0.5951 , 1.3408 , 0.2130 , - 1.3789 , 0.8363 , - 0.2101 ], [ - 1.3849 , 0.3773 , - 0.0585 , 0.6896 , - 0.0998 , 0.2804 , 0.0696 , - 0.2529 , 0.3143 , 0.3409 ], [ - 0.9103 , - 0.1578 , 1.6673 , - 0.4817 , 0.4088 , - 0.5484 , 0.6103 , - 0.2287 , - 0.0665 , 0.0055 ], [ - 1.1692 , - 2.8531 , - 1.2499 , - 0.0257 , 2.8580 , 0.2616 , - 0.7122 , - 0.0551 , 0.8112 , 2.3233 ], [ - 0.2790 , - 1.9494 , 0.6096 , - 0.5653 , 2.2792 , - 1.0687 , 0.1634 , 0.3122 , 0.1053 , 1.0884 ], [ 0.1267 , - 1.2297 , - 0.1315 , 0.2428 , - 0.5436 , 0.4123 , 2.3060 , - 0.9278 , - 0.1528 , - 0.4224 ], [ - 0.0235 , - 0.9137 , - 0.1457 , 1.6858 , - 0.7552 , 0.7293 , 0.2510 , - 0.3955 , - 0.2187 , - 0.1505 ], [ 0.5643 , - 1.2783 , - 1.4149 , 0.0304 , 0.8375 , 1.5018 , 0.0338 , - 0.3875 , - 0.0117 , 0.5751 ], [ 0.2926 , - 0.7486 , - 0.3238 , 1.0384 , 0.0308 , 0.6792 , - 0.0170 , - 0.5797 , 0.2819 , - 0.3510 ], [ 0.1219 , - 0.5862 , 1.5817 , - 0.1297 , 0.4730 , - 0.9171 , 0.7886 , - 0.7022 , - 0.0501 , - 0.2812 ], [ 1.7587 , - 2.4511 , - 0.7369 , 0.4082 , - 0.6426 , 1.1784 , 0.6052 , - 0.7178 , 1.6161 , - 0.2220 ], [ - 0.1267 , - 2.6719 , 0.0505 , - 0.4972 , 2.9027 , - 0.1461 , 0.2807 , - 0.2921 , 0.2231 , 1.1327 ], [ - 0.9892 , 2.4401 , 0.1274 , 0.2838 , - 0.7535 , - 0.1684 , - 0.6493 , - 0.1908 , 0.2290 , - 0.2150 ], [ - 0.2071 , - 2.1351 , - 0.9191 , - 0.9309 , 1.7747 , - 0.3046 , 0.0183 , 1.0136 , - 0.1016 , 2.1288 ], [ - 0.0103 , 0.3280 , - 0.6974 , - 0.2504 , 0.3187 , 0.4390 , - 0.1879 , 0.3954 , 0.2332 , - 0.1971 ], [ - 0.2280 , - 1.6754 , - 0.7438 , 0.5078 , 0.2544 , - 0.1020 , - 0.2503 , 2.0799 , - 0.5033 , 0.5890 ], [ 0.3972 , - 0.9369 , 1.2696 , - 1.6713 , - 0.4159 , - 0.0221 , 0.6489 , - 0.4777 , 1.2497 , 0.3931 ], [ - 0.7566 , - 0.8230 , - 0.0785 , - 0.3083 , 0.7821 , 0.1880 , 0.1037 , - 0.0956 , 0.4219 , 1.0798 ], [ - 1.0328 , - 0.1700 , 1.3806 , 0.5445 , - 0.2624 , - 0.0780 , - 0.3595 , - 0.6253 , 0.4309 , 0.1813 ], [ - 1.0360 , - 0.4704 , 0.1948 , - 0.7066 , 0.6600 , - 0.4633 , - 0.3602 , 1.7494 , 0.1522 , 0.6086 ], [ - 1.2032 , - 0.7903 , - 0.5754 , 0.4722 , 0.6068 , 0.5752 , 0.2151 , - 0.2495 , 0.3420 , 0.9278 ], [ 0.2247 , - 0.1361 , 0.9374 , - 0.1543 , 0.4921 , - 0.6553 , 0.5885 , 0.2617 , - 0.2216 , - 0.3736 ], [ - 0.2867 , - 1.4486 , 0.6658 , - 0.8755 , 2.3195 , - 0.7627 , - 0.2132 , 0.2488 , 0.3484 , 1.0860 ], [ - 1.4031 , - 0.4518 , - 0.3181 , 2.8268 , - 0.5371 , 1.0154 , - 0.9247 , - 0.7385 , 1.1031 , 0.0422 ], [ 2.8604 , - 1.5413 , 0.6241 , - 0.8017 , - 1.4104 , 0.6314 , 0.4614 , - 0.0218 , - 0.3411 , - 0.2609 ], [ 0.2113 , - 1.2348 , - 0.8535 , - 0.1041 , - 0.2703 , - 0.1294 , - 0.7057 , 2.7552 , - 0.4429 , 0.4517 ], [ 4.5191 , - 2.7407 , 1.1091 , 0.3975 , - 0.9456 , 1.2277 , 0.3616 , - 1.6564 , 0.5063 , - 1.4274 ], [ 1.4615 , - 1.0765 , 1.8388 , 1.5006 , - 1.2351 , 0.2781 , 0.2830 , - 0.8491 , 0.2222 , - 1.7779 ], [ - 1.2160 , 0.8502 , 0.2413 , - 0.0798 , - 0.7880 , - 0.4286 , - 0.8060 , 0.7194 , 1.2663 , 0.6412 ], [ - 1.3318 , 2.3388 , - 0.4003 , - 0.1094 , - 1.0285 , 0.1021 , - 0.0388 , - 0.0497 , 0.5137 , - 0.2507 ], [ - 1.7853 , 0.5884 , - 0.6108 , - 0.5557 , 0.8696 , - 0.6226 , - 0.7983 , 1.7169 , - 0.0145 , 0.8231 ], [ - 0.1739 , 0.1562 , - 0.2933 , 2.3195 , - 0.9480 , 1.2019 , - 0.4834 , - 1.0567 , 0.5685 , - 0.6841 ], [ - 0.7920 , - 0.3339 , 0.7452 , - 0.6529 , - 0.3307 , - 0.6092 , - 0.0950 , 1.7311 , - 0.3481 , 0.3801 ], [ - 1.7810 , 1.0676 , - 0.7611 , 0.3658 , - 0.0431 , - 0.1012 , - 0.6048 , 0.3089 , 0.9998 , 0.7164 ], [ - 0.5856 , - 0.5261 , - 0.4859 , - 1.0551 , - 0.1838 , - 0.2144 , - 1.2599 , 3.3891 , 0.4691 , 0.7566 ], [ - 0.4984 , - 1.7770 , - 1.1998 , - 0.1075 , 1.0882 , 0.4539 , - 0.5651 , 1.4381 , - 0.5678 , 1.7479 ], [ 0.2938 , - 1.8536 , 0.4259 , - 0.5429 , 0.0066 , 0.4120 , 2.3793 , - 0.3666 , - 0.2604 , 0.0382 ], [ - 0.4080 , - 0.9851 , 4.0264 , 0.1099 , - 0.1766 , - 1.1557 , 0.6419 , - 0.8147 , 0.7535 , - 1.1452 ], [ - 0.4636 , - 1.7323 , - 0.6433 , - 0.0274 , 0.7227 , - 0.1799 , - 0.9336 , 2.1881 , - 0.2073 , 1.6522 ], [ - 0.9617 , - 0.0348 , - 0.3980 , - 0.4738 , 0.7790 , 0.4671 , - 0.6115 , - 0.7067 , 1.3036 , 0.4923 ], [ - 1.0151 , - 2.5385 , - 0.6072 , 0.2902 , 3.1570 , 0.1062 , - 0.2169 , - 0.4491 , 0.6326 , 1.6829 ], [ - 1.8852 , 0.6066 , - 0.2840 , - 0.4475 , - 0.1147 , - 0.7858 , - 1.1805 , 3.0723 , 0.3960 , 0.9720 ], [ 0.0344 , - 1.4878 , - 0.9675 , 1.9649 , - 0.3146 , 1.2183 , 0.6730 , - 0.3650 , 0.0646 , - 0.0898 ], [ - 0.2118 , - 2.0350 , 0.9917 , - 0.8993 , 1.2334 , - 0.6723 , 2.5847 , - 0.0454 , - 0.4149 , 0.3927 ], [ - 1.7365 , 3.0447 , 0.5115 , 0.0786 , - 0.7544 , - 0.2158 , - 0.4876 , - 0.2891 , 0.5089 , - 0.6719 ], [ 0.3652 , - 0.5457 , - 0.1167 , 2.9056 , - 1.1622 , 0.8192 , - 1.3245 , - 0.6414 , 0.8097 , - 0.4958 ], [ - 0.8755 , - 0.6983 , 0.2208 , - 0.6463 , 0.5276 , 0.1145 , 2.7229 , - 1.0316 , 0.1905 , 0.2090 ], [ - 0.9702 , 0.1265 , - 0.0007 , - 0.5106 , 0.4970 , - 0.0804 , 0.0017 , 0.0607 , 0.6164 , 0.4490 ], [ - 0.8271 , - 0.6822 , - 0.7434 , 2.6457 , - 1.6143 , 1.1486 , - 1.0705 , 0.5611 , 0.6422 , 0.1250 ], [ - 1.9979 , 1.8175 , - 0.1658 , - 0.0343 , - 0.6292 , 0.1774 , 0.3150 , - 0.4633 , 0.9266 , 0.0252 ], [ - 0.9039 , - 0.6030 , - 0.2173 , - 1.1768 , 2.3198 , - 0.5072 , 0.3418 , - 0.1551 , 0.1282 , 1.4250 ], [ - 0.9891 , 0.5212 , - 0.4518 , 0.3267 , - 0.0759 , 0.3826 , - 0.0341 , 0.0382 , 0.2451 , 0.3658 ], [ - 2.1217 , 1.5102 , - 0.7828 , 0.3554 , - 0.4192 , - 0.0772 , 0.0578 , 0.8070 , 0.1701 , 0.5880 ], [ 1.0665 , - 1.3826 , 0.6243 , - 0.8096 , - 0.4227 , 0.5925 , 1.8112 , - 0.9946 , 0.2010 , - 0.7731 ], [ - 1.1263 , - 1.7484 , 0.0041 , - 0.5439 , 1.7242 , - 0.9475 , - 0.3835 , 0.8452 , 0.3077 , 2.2689 ]]) Printing output size This produces a 100x10 matrix because each iteration has a batch size of 100 and each prediction across the 10 classes, with the largest number indicating the likely number it is predicting. iter_test = 0 for images , labels in test_loader : iter_test += 1 images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () outputs = model ( images ) if iter_test == 1 : print ( 'OUTPUTS' ) print ( outputs . size ()) _ , predicted = torch . max ( outputs . data , 1 ) OUTPUTS torch . Size ([ 100 , 10 ]) Printing one output This would be a 1x10 matrix where the largest number is what the model thinks the image is. Here we can see that in the tensor, position 7 has the largest number, indicating the model thinks the image is 7. number 0: -0.4181 number 1: -1.0784 ... number 7: 2.9352 iter_test = 0 for images , labels in test_loader : iter_test += 1 images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () outputs = model ( images ) if iter_test == 1 : print ( 'OUTPUTS' ) print ( outputs [ 0 , :]) _ , predicted = torch . max ( outputs . data , 1 ) OUTPUTS tensor([-0.4181, -1.0784, -0.4840, -0.0985, -0.2394, -0.1801, -1.1639, 2.9352, -0.1552, 0.8852]) Printing prediction output Because our output is of size 100 (our batch size), our prediction size would also of the size 100. iter_test = 0 for images , labels in test_loader : iter_test += 1 images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () outputs = model ( images ) _ , predicted = torch . max ( outputs . data , 1 ) if iter_test == 1 : print ( 'PREDICTION' ) print ( predicted . size ()) PREDICTION torch . Size ([ 100 ]) Print prediction value We are printing our prediction which as verified above, should be digit 7. iter_test = 0 for images , labels in test_loader : iter_test += 1 images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () outputs = model ( images ) _ , predicted = torch . max ( outputs . data , 1 ) if iter_test == 1 : print ( 'PREDICTION' ) print ( predicted [ 0 ]) PREDICTION tensor ( 7 ) Print prediction, label and label size We are trying to show what we are predicting and the actual values. In this case, we're predicting the right value 7! iter_test = 0 for images , labels in test_loader : iter_test += 1 images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () outputs = model ( images ) _ , predicted = torch . max ( outputs . data , 1 ) if iter_test == 1 : print ( 'PREDICTION' ) print ( predicted [ 0 ]) print ( 'LABEL SIZE' ) print ( labels . size ()) print ( 'LABEL FOR IMAGE 0' ) print ( labels [ 0 ]) PREDICTION tensor ( 7 ) LABEL SIZE torch . Size ([ 100 ]) LABEL FOR IMAGE 0 tensor ( 7 ) Print second prediction and ground truth Again, the prediction is correct. Naturally, as our model is quite competent in this simple task. iter_test = 0 for images , labels in test_loader : iter_test += 1 images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () outputs = model ( images ) _ , predicted = torch . max ( outputs . data , 1 ) if iter_test == 1 : print ( 'PREDICTION' ) print ( predicted [ 1 ]) print ( 'LABEL SIZE' ) print ( labels . size ()) print ( 'LABEL FOR IMAGE 1' ) print ( labels [ 1 ]) PREDICTION tensor ( 2 ) LABEL SIZE torch . Size ([ 100 ]) LABEL FOR IMAGE 1 tensor ( 2 ) Print accuracy Now we know what each object represents, we can understand how we arrived at our accuracy numbers. One last thing to note is that correct.item() has this syntax is because correct is a PyTorch tensor and to get the value to compute with total which is an integer, we need to do this. correct = 0 total = 0 iter_test = 0 for images , labels in test_loader : iter_test += 1 images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () outputs = model ( images ) _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * ( correct . item () / total ) print ( accuracy ) 82.94 Explanation of Python's .sum() function Python's .sum() function allows you to do a comparison between two matrices and sum the ones that return True or in our case, those predictions that match actual labels (correct predictions). # Explaining .sum() python built-in function # correct += (predicted == labels).sum() import numpy as np a = np . ones (( 10 )) print ( a ) b = np . ones (( 10 )) print ( b ) print ( a == b ) print (( a == b ) . sum ()) # matrix a [ 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. ] # matrix b [ 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. ] # boolean array [ True True True True True True True True True True ] # number of elementswhere a matches b 10","title":"Break Down Accuracy Calculation"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#saving-model","text":"Saving PyTorch model This is how you save your model. Feel free to just change save_model = True to save your model save_model = False if save_model is True : # Saves only parameters torch . save ( model . state_dict (), 'awesome_model.pkl' )","title":"Saving Model"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#building-a-logistic-regression-model-with-pytorch-gpu","text":"CPU version The usual 7-step process, getting repetitive by now which we like. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class LogisticRegressionModel ( nn . Module ): def __init__ ( self , input_size , num_classes ): super ( LogisticRegressionModel , self ) . __init__ () self . linear = nn . Linear ( input_dim , output_dim ) def forward ( self , x ): out = self . linear ( x ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 output_dim = 10 model = LogisticRegressionModel ( input_dim , output_dim ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.001 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () labels = labels # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits # 100 x 10 outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value # 100 x 1 _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct . item () / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 1.876196026802063 . Accuracy : 64.44 Iteration : 1000. Loss : 1.5153584480285645 . Accuracy : 75.68 Iteration : 1500. Loss : 1.3521136045455933 . Accuracy : 78.98 Iteration : 2000. Loss : 1.2136967182159424 . Accuracy : 80.95 Iteration : 2500. Loss : 1.0934826135635376 . Accuracy : 81.97 Iteration : 3000. Loss : 1.024120569229126 . Accuracy : 82.49 GPU version 2 things must be on GPU - model - tensors Remember step 4 and 7 will be affected and this will be the same for all model building moving forward. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class LogisticRegressionModel ( nn . Module ): def __init__ ( self , input_size , num_classes ): super ( LogisticRegressionModel , self ) . __init__ () self . linear = nn . Linear ( input_dim , output_dim ) def forward ( self , x ): out = self . linear ( x ) return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 * 28 output_dim = 10 model = LogisticRegressionModel ( input_dim , output_dim ) ####################### # USE GPU FOR MODEL # ####################### device = torch . device ( \"cuda:0\" if torch . cuda . is_available () else \"cpu\" ) model . to ( device ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.001 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): ####################### # USE GPU FOR MODEL # ####################### images = images . view ( - 1 , 28 * 28 ) . requires_grad_ () . to ( device ) labels = labels . to ( device ) # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : ####################### # USE GPU FOR MODEL # ####################### images = images . view ( - 1 , 28 * 28 ) . to ( device ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) ####################### # USE GPU FOR MODEL # ####################### # Total correct predictions if torch . cuda . is_available (): correct += ( predicted . cpu () == labels . cpu ()) . sum () else : correct += ( predicted == labels ) . sum () accuracy = 100 * correct . item () / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 1.8571407794952393 . Accuracy : 68.99 Iteration : 1000. Loss : 1.5415704250335693 . Accuracy : 75.86 Iteration : 1500. Loss : 1.2755383253097534 . Accuracy : 78.92 Iteration : 2000. Loss : 1.2468739748001099 . Accuracy : 80.72 Iteration : 2500. Loss : 1.0708973407745361 . Accuracy : 81.73 Iteration : 3000. Loss : 1.0359245538711548 . Accuracy : 82.74","title":"Building a Logistic Regression Model with PyTorch (GPU)"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#summary","text":"We've learnt to... Success Logistic regression basics Problems of linear regression In-depth Logistic Regression Get logits Get softmax Get cross-entropy loss Aim : reduce cross-entropy loss Built a logistic regression model in CPU and GPU Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model Important things to be on GPU model tensors with gradients","title":"Summary"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#citation","text":"If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Citation"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/","text":"Long Short-Term Memory (LSTM) network with PyTorch \u00b6 Run Jupyter Notebook You can run the code for this section in this jupyter notebook link . About LSTMs: Special RNN \u00b6 Capable of learning long-term dependencies LSTM = RNN on super juice RNN Transition to LSTM \u00b6 Building an LSTM with PyTorch \u00b6 Model A: 1 Hidden Layer \u00b6 Unroll 28 time steps Each step input size: 28 x 1 Total per unroll: 28 x 28 Feedforward Neural Network input size: 28 x 28 1 Hidden layer Steps \u00b6 Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model Step 1: Loading MNIST Train Dataset \u00b6 Images from 1 to 9 The usual loading of our MNIST dataset As usual, we've 60k training images and 10k testing images. Subsequently, we'll have 3 groups: training, validation and testing for a more robust evaluation of algorithms. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) print ( train_dataset . train_data . size ()) print ( train_dataset . train_labels . size ()) print ( test_dataset . test_data . size ()) ``` python print ( test_dataset . test_labels . size ()) torch . Size ([ 60000 , 28 , 28 ]) torch . Size ([ 60000 ]) torch . Size ([ 10000 , 28 , 28 ]) torch . Size ([ 10000 ]) Step 2: Make Dataset Iterable \u00b6 Creating an iterable object for our dataset batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) Step 3: Create Model Class \u00b6 Creating an LSTM model class It is very similar to RNN in terms of the shape of our input of batch_dim x seq_dim x feature_dim . The only change is that we have our cell state on top of our hidden state. PyTorch's LSTM module handles all the other weights for our other gates. class LSTMModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , layer_dim , output_dim ): super ( LSTMModel , self ) . __init__ () # Hidden dimensions self . hidden_dim = hidden_dim # Number of hidden layers self . layer_dim = layer_dim # Building your LSTM # batch_first=True causes input/output tensors to be of shape # (batch_dim, seq_dim, feature_dim) self . lstm = nn . LSTM ( input_dim , hidden_dim , layer_dim , batch_first = True ) # Readout layer self . fc = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Initialize hidden state with zeros h0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () # Initialize cell state c0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () # 28 time steps # We need to detach as we are doing truncated backpropagation through time (BPTT) # If we don't, we'll backprop all the way to the start even after going through another batch out , ( hn , cn ) = self . lstm ( x , ( h0 . detach (), c0 . detach ())) # Index hidden state of last time step # out.size() --> 100, 28, 100 # out[:, -1, :] --> 100, 100 --> just want last time step hidden states! out = self . fc ( out [:, - 1 , :]) # out.size() --> 100, 10 return out Step 4: Instantiate Model Class \u00b6 28 time steps Each time step: input dimension = 28 1 hidden layer MNIST 1-9 digits \\(\\rightarrow\\) output dimension = 10 Instantiate our LSTM model input_dim = 28 hidden_dim = 100 layer_dim = 1 output_dim = 10 model = LSTMModel ( input_dim , hidden_dim , layer_dim , output_dim ) Step 5: Instantiate Loss Class \u00b6 Long Short-Term Memory Neural Network: Cross Entropy Loss Recurrent Neural Network : Cross Entropy Loss Convolutional Neural Network : Cross Entropy Loss Feedforward Neural Network : Cross Entropy Loss Logistic Regression : Cross Entropy Loss Linear Regression : MSE Cross Entry Loss Function Because we are doing a classification problem we'll be using a Cross Entropy function. If we were to do a regression problem, then we would typically use a MSE function. criterion = nn . CrossEntropyLoss () Step 6: Instantiate Optimizer Class \u00b6 Simplified equation \\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\) \\(\\theta\\) : parameters (our variables) \\(\\eta\\) : learning rate (how fast we want to learn) \\(\\nabla_\\theta\\) : parameters' gradients Even simplier equation parameters = parameters - learning_rate * parameters_gradients At every iteration, we update our model's parameters Mini-batch Stochastic Gradient Descent learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) Parameters In-Depth \u00b6 1 Layer LSTM Groups of Parameters We will have 6 groups of parameters here comprising weights and biases from: - Input to Hidden Layer Affine Function - Hidden Layer to Output Affine Function - Hidden Layer to Hidden Layer Affine Function Notice how this is exactly the same number of groups of parameters as our RNN? But the sizes of these groups will be larger for an LSTM due to its gates. len ( list ( model . parameters ())) 6 In-depth Parameters Analysis Comparing to RNN's parameters , we've the same number of groups but for LSTM we've 4x the number of parameters! for i in range ( len ( list ( model . parameters ()))): print ( list ( model . parameters ())[ i ] . size ()) torch . Size ([ 400 , 28 ]) torch . Size ([ 400 , 100 ]) torch . Size ([ 400 ]) torch . Size ([ 400 ]) torch . Size ([ 10 , 100 ]) torch . Size ([ 10 ]) Parameters Breakdown \u00b6 This is the breakdown of the parameters associated with the respective affine functions Input \\(\\rightarrow\\) Gates \\([400, 28] \\rightarrow w_1, w_3, w_5, w_7\\) \\([400] \\rightarrow b_1, b_3, b_5, b_7\\) Hidden State \\(\\rightarrow\\) Gates \\([400,100] \\rightarrow w_2, w_4, w_6, w_8\\) \\([400] \\rightarrow b_2, b_4, b_6, b_8\\) Hidden State \\(\\rightarrow\\) Output \\([10, 100] \\rightarrow w_9\\) \\([10] \\rightarrow b_9\\) Step 7: Train Model \u00b6 Process Convert inputs/labels to variables LSTM Input: (1, 28) RNN Input: (1, 28) CNN Input: (1, 28, 28) FNN Input: (1, 28*28) Clear gradient buffets Get output given inputs Get loss Get gradients w.r.t. parameters Update parameters using gradients parameters = parameters - learning_rate * parameters_gradients REPEAT Training 1 Hidden Layer LSTM # Number of steps to unroll seq_dim = 28 iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as a torch tensor with gradient accumulation abilities images = images . view ( - 1 , seq_dim , input_dim ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits # outputs.size() --> 100, 10 outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Resize images images = images . view ( - 1 , seq_dim , input_dim ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 0.8390830755233765 . Accuracy : 72 Iteration : 1000. Loss : 0.46470555663108826 . Accuracy : 85 Iteration : 1500. Loss : 0.31465113162994385 . Accuracy : 91 Iteration : 2000. Loss : 0.19143860042095184 . Accuracy : 94 Iteration : 2500. Loss : 0.16134005784988403 . Accuracy : 95 Iteration : 3000. Loss : 0.255976140499115 . Accuracy : 95 Model B: 2 Hidden Layer \u00b6 Unroll 28 time steps Each step input size: 28 x 1 Total per unroll: 28 x 28 Feedforward Neural Network inpt size: 28 x 28 2 Hidden layer Steps \u00b6 Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model Train 2 Hidden Layer LSTM import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class LSTMModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , layer_dim , output_dim ): super ( LSTMModel , self ) . __init__ () # Hidden dimensions self . hidden_dim = hidden_dim # Number of hidden layers self . layer_dim = layer_dim # Building your LSTM # batch_first=True causes input/output tensors to be of shape # (batch_dim, seq_dim, feature_dim) self . lstm = nn . LSTM ( input_dim , hidden_dim , layer_dim , batch_first = True ) # Readout layer self . fc = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Initialize hidden state with zeros h0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () # Initialize cell state c0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () # One time step # We need to detach as we are doing truncated backpropagation through time (BPTT) # If we don't, we'll backprop all the way to the start even after going through another batch out , ( hn , cn ) = self . lstm ( x , ( h0 . detach (), c0 . detach ())) # Index hidden state of last time step # out.size() --> 100, 28, 100 # out[:, -1, :] --> 100, 100 --> just want last time step hidden states! out = self . fc ( out [:, - 1 , :]) # out.size() --> 100, 10 return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 hidden_dim = 100 layer_dim = 2 # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER output_dim = 10 model = LSTMModel ( input_dim , hidden_dim , layer_dim , output_dim ) # JUST PRINTING MODEL & PARAMETERS print ( model ) print ( len ( list ( model . parameters ()))) for i in range ( len ( list ( model . parameters ()))): print ( list ( model . parameters ())[ i ] . size ()) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' # Number of steps to unroll seq_dim = 28 iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as torch tensor with gradient accumulation abilities images = images . view ( - 1 , seq_dim , input_dim ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits # outputs.size() --> 100, 10 outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Resize image images = images . view ( - 1 , seq_dim , input_dim ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) LSTMModel ( ( lstm ): LSTM ( 28 , 100 , num_layers = 2 , batch_first = True ) ( fc ): Linear ( in_features = 100 , out_features = 10 , bias = True ) ) 10 torch . Size ([ 400 , 28 ]) torch . Size ([ 400 , 100 ]) torch . Size ([ 400 ]) torch . Size ([ 400 ]) torch . Size ([ 400 , 100 ]) torch . Size ([ 400 , 100 ]) torch . Size ([ 400 ]) torch . Size ([ 400 ]) torch . Size ([ 10 , 100 ]) torch . Size ([ 10 ]) Iteration : 500. Loss : 2.3074915409088135 . Accuracy : 11 Iteration : 1000. Loss : 1.8854578733444214 . Accuracy : 35 Iteration : 1500. Loss : 0.5317062139511108 . Accuracy : 80 Iteration : 2000. Loss : 0.15290376543998718 . Accuracy : 92 Iteration : 2500. Loss : 0.19500978291034698 . Accuracy : 93 Iteration : 3000. Loss : 0.10683634132146835 . Accuracy : 95 Parameters Breakdown (Layer 1) \u00b6 Input \\(\\rightarrow\\) Gates \\([400, 28]\\) \\([400]\\) Hidden State \\(\\rightarrow\\) Gates \\([400,100]\\) \\([400]\\) Parameters Breakdown (Layer 2) \u00b6 Input \\(\\rightarrow\\) Gates \\([400, 100]\\) \\([400]\\) Hidden State \\(\\rightarrow\\) Gates \\([400,100]\\) \\([400]\\) Parameters Breakdown (Readout Layer) \u00b6 Hidden State \\(\\rightarrow\\) Output \\([10, 100]\\) \\([10]\\) Model C: 3 Hidden Layer \u00b6 Unroll 28 time steps Each step input size: 28 x 1 Total per unroll: 28 x 28 Feedforward Neural Network inpt size: 28 x 28 3 Hidden layer Steps \u00b6 Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 3 Hidden Layer LSTM import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class LSTMModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , layer_dim , output_dim ): super ( LSTMModel , self ) . __init__ () # Hidden dimensions self . hidden_dim = hidden_dim # Number of hidden layers self . layer_dim = layer_dim # Building your LSTM # batch_first=True causes input/output tensors to be of shape # (batch_dim, seq_dim, feature_dim) self . lstm = nn . LSTM ( input_dim , hidden_dim , layer_dim , batch_first = True ) # Readout layer self . fc = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Initialize hidden state with zeros h0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () # Initialize cell state c0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () # One time step # We need to detach as we are doing truncated backpropagation through time (BPTT) # If we don't, we'll backprop all the way to the start even after going through another batch out , ( hn , cn ) = self . lstm ( x , ( h0 . detach (), c0 . detach ())) # Index hidden state of last time step # out.size() --> 100, 28, 100 # out[:, -1, :] --> 100, 100 --> just want last time step hidden states! out = self . fc ( out [:, - 1 , :]) # out.size() --> 100, 10 return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 hidden_dim = 100 layer_dim = 3 # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER output_dim = 10 model = LSTMModel ( input_dim , hidden_dim , layer_dim , output_dim ) # JUST PRINTING MODEL & PARAMETERS print ( model ) print ( len ( list ( model . parameters ()))) for i in range ( len ( list ( model . parameters ()))): print ( list ( model . parameters ())[ i ] . size ()) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' # Number of steps to unroll seq_dim = 28 iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , seq_dim , input_dim ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits # outputs.size() --> 100, 10 outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , seq_dim , input_dim ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) LSTMModel( (lstm): LSTM(28, 100, num_layers=3, batch_first=True) (fc): Linear(in_features=100, out_features=10, bias=True) ) 14 torch.Size([400, 28]) torch.Size([400, 100]) torch.Size([400]) torch.Size([400]) torch.Size([400, 100]) torch.Size([400, 100]) torch.Size([400]) torch.Size([400]) torch.Size([400, 100]) torch.Size([400, 100]) torch.Size([400]) torch.Size([400]) torch.Size([10, 100]) torch.Size([10]) Iteration: 500. Loss: 2.2927396297454834. Accuracy: 11 Iteration: 1000. Loss: 2.29740309715271. Accuracy: 11 Iteration: 1500. Loss: 2.1950502395629883. Accuracy: 20 Iteration: 2000. Loss: 1.0738657712936401. Accuracy: 59 Iteration: 2500. Loss: 0.5988132357597351. Accuracy: 79 Iteration: 3000. Loss: 0.4107239246368408. Accuracy: 88 Parameters Breakdown (Layer 1) \u00b6 Input \\(\\rightarrow\\) Gates [400, 28] [400] Hidden State \\(\\rightarrow\\) Gates [400,100] [400] Parameters Breakdown (Layer 2) \u00b6 Input \\(\\rightarrow\\) Gates [400, 100] [400] Hidden State \\(\\rightarrow\\) Gates [400,100] [400] Parameters Breakdown (Layer 3) \u00b6 Input \\(\\rightarrow\\) Gates [400, 100] [400] Hidden State \\(\\rightarrow\\) Gates [400,100] [400] Parameters Breakdown (Readout Layer) \u00b6 Hidden State \\(\\rightarrow\\) Output [10, 100] [10] Comparison with RNN \u00b6 Model A RNN Model B RNN Model C RNN ReLU ReLU Tanh 1 Hidden Layer 2 Hidden Layers 3 Hidden Layers 100 Hidden Units 100 Hidden Units 100 Hidden Units 92.48% 95.09% 95.54% Model A LSTM Model B LSTM Model C LSTM 1 Hidden Layer 2 Hidden Layers 3 Hidden Layers 100 Hidden Units 100 Hidden Units 100 Hidden Units 96.05% 95.24% 91.22% Deep Learning Notes \u00b6 2 ways to expand a recurrent neural network More hidden units (o, i, f, g) gates More hidden layers Cons Need a larger dataset Curse of dimensionality Does not necessarily mean higher accuracy 3. Building a Recurrent Neural Network with PyTorch (GPU) \u00b6 Model A: 3 Hidden Layers \u00b6 GPU: 2 things must be on GPU - model - tensors Steps \u00b6 Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 3 Hidden Layer LSTM on GPU import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class LSTMModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , layer_dim , output_dim ): super ( LSTMModel , self ) . __init__ () # Hidden dimensions self . hidden_dim = hidden_dim # Number of hidden layers self . layer_dim = layer_dim # Building your LSTM # batch_first=True causes input/output tensors to be of shape # (batch_dim, seq_dim, feature_dim) self . lstm = nn . LSTM ( input_dim , hidden_dim , layer_dim , batch_first = True ) # Readout layer self . fc = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Initialize hidden state with zeros ####################### # USE GPU FOR MODEL # ####################### h0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () . to ( device ) # Initialize cell state c0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () . to ( device ) # One time step out , ( hn , cn ) = self . lstm ( x , ( h0 . detach (), c0 . detach ())) # Index hidden state of last time step # out.size() --> 100, 28, 100 # out[:, -1, :] --> 100, 100 --> just want last time step hidden states! out = self . fc ( out [:, - 1 , :]) # out.size() --> 100, 10 return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 hidden_dim = 100 layer_dim = 3 # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER output_dim = 10 model = LSTMModel ( input_dim , hidden_dim , layer_dim , output_dim ) ####################### # USE GPU FOR MODEL # ####################### device = torch . device ( \"cuda:0\" if torch . cuda . is_available () else \"cpu\" ) model . to ( device ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' # Number of steps to unroll seq_dim = 28 iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable ####################### # USE GPU FOR MODEL # ####################### images = images . view ( - 1 , seq_dim , input_dim ) . requires_grad_ () . to ( device ) labels = labels . to ( device ) # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits # outputs.size() --> 100, 10 outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : ####################### # USE GPU FOR MODEL # ####################### images = images . view ( - 1 , seq_dim , input_dim ) . to ( device ) labels = labels . to ( device ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions ####################### # USE GPU FOR MODEL # ####################### if torch . cuda . is_available (): correct += ( predicted . cpu () == labels . cpu ()) . sum () else : correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 2.3068575859069824 . Accuracy : 11 Iteration : 1000. Loss : 2.291989803314209 . Accuracy : 14 Iteration : 1500. Loss : 1.909593105316162 . Accuracy : 28 Iteration : 2000. Loss : 0.7345633506774902 . Accuracy : 71 Iteration : 2500. Loss : 0.45030108094215393 . Accuracy : 86 Iteration : 3000. Loss : 0.2627193331718445 . Accuracy : 89 Summary \u00b6 We've learnt to... Success RNN transition to LSTM LSTM Models in PyTorch Model A: 1 Hidden Layer LSTM Model B: 2 Hidden Layer LSTM Model C: 3 Hidden Layer LSTM Models Variation in Code Modifying only step 4 Ways to Expand Model\u2019s Capacity More hidden units More hidden layers Cons of Expanding Capacity Need more data Does not necessarily mean higher accuracy GPU Code 2 things on GPU model tensors Modifying only Step 3, 4 and 7 7 Step Model Building Recap Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model Citation \u00b6 If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Long Short Term Memory Neural Networks (LSTM)"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#long-short-term-memory-lstm-network-with-pytorch","text":"Run Jupyter Notebook You can run the code for this section in this jupyter notebook link .","title":"Long Short-Term Memory (LSTM) network with PyTorch"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#about-lstms-special-rnn","text":"Capable of learning long-term dependencies LSTM = RNN on super juice","title":"About LSTMs: Special RNN"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#rnn-transition-to-lstm","text":"","title":"RNN Transition to LSTM"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#building-an-lstm-with-pytorch","text":"","title":"Building an LSTM with PyTorch"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#model-a-1-hidden-layer","text":"Unroll 28 time steps Each step input size: 28 x 1 Total per unroll: 28 x 28 Feedforward Neural Network input size: 28 x 28 1 Hidden layer","title":"Model A: 1 Hidden Layer"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#steps","text":"Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model","title":"Steps"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#step-1-loading-mnist-train-dataset","text":"Images from 1 to 9 The usual loading of our MNIST dataset As usual, we've 60k training images and 10k testing images. Subsequently, we'll have 3 groups: training, validation and testing for a more robust evaluation of algorithms. import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) print ( train_dataset . train_data . size ()) print ( train_dataset . train_labels . size ()) print ( test_dataset . test_data . size ()) ``` python print ( test_dataset . test_labels . size ()) torch . Size ([ 60000 , 28 , 28 ]) torch . Size ([ 60000 ]) torch . Size ([ 10000 , 28 , 28 ]) torch . Size ([ 10000 ])","title":"Step 1: Loading MNIST Train Dataset"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#step-2-make-dataset-iterable","text":"Creating an iterable object for our dataset batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False )","title":"Step 2: Make Dataset Iterable"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#step-3-create-model-class","text":"Creating an LSTM model class It is very similar to RNN in terms of the shape of our input of batch_dim x seq_dim x feature_dim . The only change is that we have our cell state on top of our hidden state. PyTorch's LSTM module handles all the other weights for our other gates. class LSTMModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , layer_dim , output_dim ): super ( LSTMModel , self ) . __init__ () # Hidden dimensions self . hidden_dim = hidden_dim # Number of hidden layers self . layer_dim = layer_dim # Building your LSTM # batch_first=True causes input/output tensors to be of shape # (batch_dim, seq_dim, feature_dim) self . lstm = nn . LSTM ( input_dim , hidden_dim , layer_dim , batch_first = True ) # Readout layer self . fc = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Initialize hidden state with zeros h0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () # Initialize cell state c0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () # 28 time steps # We need to detach as we are doing truncated backpropagation through time (BPTT) # If we don't, we'll backprop all the way to the start even after going through another batch out , ( hn , cn ) = self . lstm ( x , ( h0 . detach (), c0 . detach ())) # Index hidden state of last time step # out.size() --> 100, 28, 100 # out[:, -1, :] --> 100, 100 --> just want last time step hidden states! out = self . fc ( out [:, - 1 , :]) # out.size() --> 100, 10 return out","title":"Step 3: Create Model Class"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#step-4-instantiate-model-class","text":"28 time steps Each time step: input dimension = 28 1 hidden layer MNIST 1-9 digits \\(\\rightarrow\\) output dimension = 10 Instantiate our LSTM model input_dim = 28 hidden_dim = 100 layer_dim = 1 output_dim = 10 model = LSTMModel ( input_dim , hidden_dim , layer_dim , output_dim )","title":"Step 4: Instantiate Model Class"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#step-5-instantiate-loss-class","text":"Long Short-Term Memory Neural Network: Cross Entropy Loss Recurrent Neural Network : Cross Entropy Loss Convolutional Neural Network : Cross Entropy Loss Feedforward Neural Network : Cross Entropy Loss Logistic Regression : Cross Entropy Loss Linear Regression : MSE Cross Entry Loss Function Because we are doing a classification problem we'll be using a Cross Entropy function. If we were to do a regression problem, then we would typically use a MSE function. criterion = nn . CrossEntropyLoss ()","title":"Step 5: Instantiate Loss Class"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#step-6-instantiate-optimizer-class","text":"Simplified equation \\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\) \\(\\theta\\) : parameters (our variables) \\(\\eta\\) : learning rate (how fast we want to learn) \\(\\nabla_\\theta\\) : parameters' gradients Even simplier equation parameters = parameters - learning_rate * parameters_gradients At every iteration, we update our model's parameters Mini-batch Stochastic Gradient Descent learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate )","title":"Step 6: Instantiate Optimizer Class"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#parameters-in-depth","text":"1 Layer LSTM Groups of Parameters We will have 6 groups of parameters here comprising weights and biases from: - Input to Hidden Layer Affine Function - Hidden Layer to Output Affine Function - Hidden Layer to Hidden Layer Affine Function Notice how this is exactly the same number of groups of parameters as our RNN? But the sizes of these groups will be larger for an LSTM due to its gates. len ( list ( model . parameters ())) 6 In-depth Parameters Analysis Comparing to RNN's parameters , we've the same number of groups but for LSTM we've 4x the number of parameters! for i in range ( len ( list ( model . parameters ()))): print ( list ( model . parameters ())[ i ] . size ()) torch . Size ([ 400 , 28 ]) torch . Size ([ 400 , 100 ]) torch . Size ([ 400 ]) torch . Size ([ 400 ]) torch . Size ([ 10 , 100 ]) torch . Size ([ 10 ])","title":"Parameters In-Depth"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#parameters-breakdown","text":"This is the breakdown of the parameters associated with the respective affine functions Input \\(\\rightarrow\\) Gates \\([400, 28] \\rightarrow w_1, w_3, w_5, w_7\\) \\([400] \\rightarrow b_1, b_3, b_5, b_7\\) Hidden State \\(\\rightarrow\\) Gates \\([400,100] \\rightarrow w_2, w_4, w_6, w_8\\) \\([400] \\rightarrow b_2, b_4, b_6, b_8\\) Hidden State \\(\\rightarrow\\) Output \\([10, 100] \\rightarrow w_9\\) \\([10] \\rightarrow b_9\\)","title":"Parameters Breakdown"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#step-7-train-model","text":"Process Convert inputs/labels to variables LSTM Input: (1, 28) RNN Input: (1, 28) CNN Input: (1, 28, 28) FNN Input: (1, 28*28) Clear gradient buffets Get output given inputs Get loss Get gradients w.r.t. parameters Update parameters using gradients parameters = parameters - learning_rate * parameters_gradients REPEAT Training 1 Hidden Layer LSTM # Number of steps to unroll seq_dim = 28 iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as a torch tensor with gradient accumulation abilities images = images . view ( - 1 , seq_dim , input_dim ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits # outputs.size() --> 100, 10 outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Resize images images = images . view ( - 1 , seq_dim , input_dim ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 0.8390830755233765 . Accuracy : 72 Iteration : 1000. Loss : 0.46470555663108826 . Accuracy : 85 Iteration : 1500. Loss : 0.31465113162994385 . Accuracy : 91 Iteration : 2000. Loss : 0.19143860042095184 . Accuracy : 94 Iteration : 2500. Loss : 0.16134005784988403 . Accuracy : 95 Iteration : 3000. Loss : 0.255976140499115 . Accuracy : 95","title":"Step 7: Train Model"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#model-b-2-hidden-layer","text":"Unroll 28 time steps Each step input size: 28 x 1 Total per unroll: 28 x 28 Feedforward Neural Network inpt size: 28 x 28 2 Hidden layer","title":"Model B: 2 Hidden Layer"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#steps_1","text":"Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model Train 2 Hidden Layer LSTM import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class LSTMModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , layer_dim , output_dim ): super ( LSTMModel , self ) . __init__ () # Hidden dimensions self . hidden_dim = hidden_dim # Number of hidden layers self . layer_dim = layer_dim # Building your LSTM # batch_first=True causes input/output tensors to be of shape # (batch_dim, seq_dim, feature_dim) self . lstm = nn . LSTM ( input_dim , hidden_dim , layer_dim , batch_first = True ) # Readout layer self . fc = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Initialize hidden state with zeros h0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () # Initialize cell state c0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () # One time step # We need to detach as we are doing truncated backpropagation through time (BPTT) # If we don't, we'll backprop all the way to the start even after going through another batch out , ( hn , cn ) = self . lstm ( x , ( h0 . detach (), c0 . detach ())) # Index hidden state of last time step # out.size() --> 100, 28, 100 # out[:, -1, :] --> 100, 100 --> just want last time step hidden states! out = self . fc ( out [:, - 1 , :]) # out.size() --> 100, 10 return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 hidden_dim = 100 layer_dim = 2 # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER output_dim = 10 model = LSTMModel ( input_dim , hidden_dim , layer_dim , output_dim ) # JUST PRINTING MODEL & PARAMETERS print ( model ) print ( len ( list ( model . parameters ()))) for i in range ( len ( list ( model . parameters ()))): print ( list ( model . parameters ())[ i ] . size ()) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' # Number of steps to unroll seq_dim = 28 iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as torch tensor with gradient accumulation abilities images = images . view ( - 1 , seq_dim , input_dim ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits # outputs.size() --> 100, 10 outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Resize image images = images . view ( - 1 , seq_dim , input_dim ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) LSTMModel ( ( lstm ): LSTM ( 28 , 100 , num_layers = 2 , batch_first = True ) ( fc ): Linear ( in_features = 100 , out_features = 10 , bias = True ) ) 10 torch . Size ([ 400 , 28 ]) torch . Size ([ 400 , 100 ]) torch . Size ([ 400 ]) torch . Size ([ 400 ]) torch . Size ([ 400 , 100 ]) torch . Size ([ 400 , 100 ]) torch . Size ([ 400 ]) torch . Size ([ 400 ]) torch . Size ([ 10 , 100 ]) torch . Size ([ 10 ]) Iteration : 500. Loss : 2.3074915409088135 . Accuracy : 11 Iteration : 1000. Loss : 1.8854578733444214 . Accuracy : 35 Iteration : 1500. Loss : 0.5317062139511108 . Accuracy : 80 Iteration : 2000. Loss : 0.15290376543998718 . Accuracy : 92 Iteration : 2500. Loss : 0.19500978291034698 . Accuracy : 93 Iteration : 3000. Loss : 0.10683634132146835 . Accuracy : 95","title":"Steps"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#parameters-breakdown-layer-1","text":"Input \\(\\rightarrow\\) Gates \\([400, 28]\\) \\([400]\\) Hidden State \\(\\rightarrow\\) Gates \\([400,100]\\) \\([400]\\)","title":"Parameters Breakdown (Layer 1)"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#parameters-breakdown-layer-2","text":"Input \\(\\rightarrow\\) Gates \\([400, 100]\\) \\([400]\\) Hidden State \\(\\rightarrow\\) Gates \\([400,100]\\) \\([400]\\)","title":"Parameters Breakdown (Layer 2)"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#parameters-breakdown-readout-layer","text":"Hidden State \\(\\rightarrow\\) Output \\([10, 100]\\) \\([10]\\)","title":"Parameters Breakdown (Readout Layer)"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#model-c-3-hidden-layer","text":"Unroll 28 time steps Each step input size: 28 x 1 Total per unroll: 28 x 28 Feedforward Neural Network inpt size: 28 x 28 3 Hidden layer","title":"Model C: 3 Hidden Layer"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#steps_2","text":"Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 3 Hidden Layer LSTM import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class LSTMModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , layer_dim , output_dim ): super ( LSTMModel , self ) . __init__ () # Hidden dimensions self . hidden_dim = hidden_dim # Number of hidden layers self . layer_dim = layer_dim # Building your LSTM # batch_first=True causes input/output tensors to be of shape # (batch_dim, seq_dim, feature_dim) self . lstm = nn . LSTM ( input_dim , hidden_dim , layer_dim , batch_first = True ) # Readout layer self . fc = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Initialize hidden state with zeros h0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () # Initialize cell state c0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () # One time step # We need to detach as we are doing truncated backpropagation through time (BPTT) # If we don't, we'll backprop all the way to the start even after going through another batch out , ( hn , cn ) = self . lstm ( x , ( h0 . detach (), c0 . detach ())) # Index hidden state of last time step # out.size() --> 100, 28, 100 # out[:, -1, :] --> 100, 100 --> just want last time step hidden states! out = self . fc ( out [:, - 1 , :]) # out.size() --> 100, 10 return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 hidden_dim = 100 layer_dim = 3 # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER output_dim = 10 model = LSTMModel ( input_dim , hidden_dim , layer_dim , output_dim ) # JUST PRINTING MODEL & PARAMETERS print ( model ) print ( len ( list ( model . parameters ()))) for i in range ( len ( list ( model . parameters ()))): print ( list ( model . parameters ())[ i ] . size ()) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' # Number of steps to unroll seq_dim = 28 iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable images = images . view ( - 1 , seq_dim , input_dim ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits # outputs.size() --> 100, 10 outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch Variable images = images . view ( - 1 , seq_dim , input_dim ) . requires_grad_ () # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) LSTMModel( (lstm): LSTM(28, 100, num_layers=3, batch_first=True) (fc): Linear(in_features=100, out_features=10, bias=True) ) 14 torch.Size([400, 28]) torch.Size([400, 100]) torch.Size([400]) torch.Size([400]) torch.Size([400, 100]) torch.Size([400, 100]) torch.Size([400]) torch.Size([400]) torch.Size([400, 100]) torch.Size([400, 100]) torch.Size([400]) torch.Size([400]) torch.Size([10, 100]) torch.Size([10]) Iteration: 500. Loss: 2.2927396297454834. Accuracy: 11 Iteration: 1000. Loss: 2.29740309715271. Accuracy: 11 Iteration: 1500. Loss: 2.1950502395629883. Accuracy: 20 Iteration: 2000. Loss: 1.0738657712936401. Accuracy: 59 Iteration: 2500. Loss: 0.5988132357597351. Accuracy: 79 Iteration: 3000. Loss: 0.4107239246368408. Accuracy: 88","title":"Steps"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#parameters-breakdown-layer-1_1","text":"Input \\(\\rightarrow\\) Gates [400, 28] [400] Hidden State \\(\\rightarrow\\) Gates [400,100] [400]","title":"Parameters Breakdown (Layer 1)"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#parameters-breakdown-layer-2_1","text":"Input \\(\\rightarrow\\) Gates [400, 100] [400] Hidden State \\(\\rightarrow\\) Gates [400,100] [400]","title":"Parameters Breakdown (Layer 2)"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#parameters-breakdown-layer-3","text":"Input \\(\\rightarrow\\) Gates [400, 100] [400] Hidden State \\(\\rightarrow\\) Gates [400,100] [400]","title":"Parameters Breakdown (Layer 3)"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#parameters-breakdown-readout-layer_1","text":"Hidden State \\(\\rightarrow\\) Output [10, 100] [10]","title":"Parameters Breakdown (Readout Layer)"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#comparison-with-rnn","text":"Model A RNN Model B RNN Model C RNN ReLU ReLU Tanh 1 Hidden Layer 2 Hidden Layers 3 Hidden Layers 100 Hidden Units 100 Hidden Units 100 Hidden Units 92.48% 95.09% 95.54% Model A LSTM Model B LSTM Model C LSTM 1 Hidden Layer 2 Hidden Layers 3 Hidden Layers 100 Hidden Units 100 Hidden Units 100 Hidden Units 96.05% 95.24% 91.22%","title":"Comparison with RNN"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#deep-learning-notes","text":"2 ways to expand a recurrent neural network More hidden units (o, i, f, g) gates More hidden layers Cons Need a larger dataset Curse of dimensionality Does not necessarily mean higher accuracy","title":"Deep Learning Notes"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#3-building-a-recurrent-neural-network-with-pytorch-gpu","text":"","title":"3. Building a Recurrent Neural Network with PyTorch (GPU)"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#model-a-3-hidden-layers","text":"GPU: 2 things must be on GPU - model - tensors","title":"Model A: 3 Hidden Layers"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#steps_3","text":"Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 3 Hidden Layer LSTM on GPU import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class LSTMModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , layer_dim , output_dim ): super ( LSTMModel , self ) . __init__ () # Hidden dimensions self . hidden_dim = hidden_dim # Number of hidden layers self . layer_dim = layer_dim # Building your LSTM # batch_first=True causes input/output tensors to be of shape # (batch_dim, seq_dim, feature_dim) self . lstm = nn . LSTM ( input_dim , hidden_dim , layer_dim , batch_first = True ) # Readout layer self . fc = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Initialize hidden state with zeros ####################### # USE GPU FOR MODEL # ####################### h0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () . to ( device ) # Initialize cell state c0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () . to ( device ) # One time step out , ( hn , cn ) = self . lstm ( x , ( h0 . detach (), c0 . detach ())) # Index hidden state of last time step # out.size() --> 100, 28, 100 # out[:, -1, :] --> 100, 100 --> just want last time step hidden states! out = self . fc ( out [:, - 1 , :]) # out.size() --> 100, 10 return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 hidden_dim = 100 layer_dim = 3 # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER output_dim = 10 model = LSTMModel ( input_dim , hidden_dim , layer_dim , output_dim ) ####################### # USE GPU FOR MODEL # ####################### device = torch . device ( \"cuda:0\" if torch . cuda . is_available () else \"cpu\" ) model . to ( device ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' # Number of steps to unroll seq_dim = 28 iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as Variable ####################### # USE GPU FOR MODEL # ####################### images = images . view ( - 1 , seq_dim , input_dim ) . requires_grad_ () . to ( device ) labels = labels . to ( device ) # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits # outputs.size() --> 100, 10 outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : ####################### # USE GPU FOR MODEL # ####################### images = images . view ( - 1 , seq_dim , input_dim ) . to ( device ) labels = labels . to ( device ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions ####################### # USE GPU FOR MODEL # ####################### if torch . cuda . is_available (): correct += ( predicted . cpu () == labels . cpu ()) . sum () else : correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 2.3068575859069824 . Accuracy : 11 Iteration : 1000. Loss : 2.291989803314209 . Accuracy : 14 Iteration : 1500. Loss : 1.909593105316162 . Accuracy : 28 Iteration : 2000. Loss : 0.7345633506774902 . Accuracy : 71 Iteration : 2500. Loss : 0.45030108094215393 . Accuracy : 86 Iteration : 3000. Loss : 0.2627193331718445 . Accuracy : 89","title":"Steps"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#summary","text":"We've learnt to... Success RNN transition to LSTM LSTM Models in PyTorch Model A: 1 Hidden Layer LSTM Model B: 2 Hidden Layer LSTM Model C: 3 Hidden Layer LSTM Models Variation in Code Modifying only step 4 Ways to Expand Model\u2019s Capacity More hidden units More hidden layers Cons of Expanding Capacity Need more data Does not necessarily mean higher accuracy GPU Code 2 things on GPU model tensors Modifying only Step 3, 4 and 7 7 Step Model Building Recap Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model","title":"Summary"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#citation","text":"If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Citation"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/","text":"Matrices with PyTorch \u00b6 Run Jupyter Notebook You can run the code for this section in this jupyter notebook link . Matrices \u00b6 Matrices Brief Introduction \u00b6 Basic definition: rectangular array of numbers. Tensors (PyTorch) Ndarrays (NumPy) 2 x 2 Matrix (R x C) 1 1 1 1 2 x 3 Matrix 1 1 1 1 1 1 Creating Matrices \u00b6 Create list # Creating a 2x2 array arr = [[ 1 , 2 ], [ 3 , 4 ]] print ( arr ) [[ 1 , 2 ], [ 3 , 4 ]] Create numpy array via list import numpy as np # Convert to NumPy np . array ( arr ) array ([[ 1 , 2 ], [ 3 , 4 ]]) Convert numpy array to PyTorch tensor import torch # Convert to PyTorch Tensor torch . Tensor ( arr ) 1 2 3 4 [ torch . FloatTensor of size 2 x2 ] Create Matrices with Default Values \u00b6 Create 2x2 numpy array of 1's np . ones (( 2 , 2 )) array ([[ 1. , 1. ], [ 1. , 1. ]]) Create 2x2 torch tensor of 1's torch . ones (( 2 , 2 )) 1 1 1 1 [torch.FloatTensor of size 2x2] Create 2x2 numpy array of random numbers np . random . rand ( 2 , 2 ) array ([[ 0.68270631 , 0.87721678 ], [ 0.07420986 , 0.79669375 ]]) Create 2x2 PyTorch tensor of random numbers torch . rand ( 2 , 2 ) 0.3900 0.8268 0.3888 0.5914 [ torch . FloatTensor of size 2 x2 ] Seeds for Reproducibility \u00b6 Why do we need seeds? We need seeds to enable reproduction of experimental results. This becomes critical later on where you can easily let people reproduce your code's output exactly as you've produced. Create seed to enable fixed numbers for random number generation # Seed np . random . seed ( 0 ) np . random . rand ( 2 , 2 ) array ([[ 0.5488135 , 0.71518937 ], [ 0.60276338 , 0.54488318 ]]) Repeat random array generation to check If you do not set the seed, you would not get the same set of numbers like here. # Seed np . random . seed ( 0 ) np . random . rand ( 2 , 2 ) array ([[ 0.5488135 , 0.71518937 ], [ 0.60276338 , 0.54488318 ]]) Create a numpy array without seed Notice how you get different numbers compared to the first 2 tries? # No seed np . random . rand ( 2 , 2 ) array ([[ 0.56804456 , 0.92559664 ], [ 0.07103606 , 0.0871293 ]]) Repeat numpy array generation without seed You get the point now, you get a totally different set of numbers. # No seed np . random . rand ( 2 , 2 ) array ([[ 0.0202184 , 0.83261985 ], [ 0.77815675 , 0.87001215 ]]) Create a PyTorch tensor with a fixed seed # Torch Seed torch . manual_seed ( 0 ) torch . rand ( 2 , 2 ) 0.5488 0.5928 0.7152 0.8443 [ torch . FloatTensor of size 2 x2 ] Repeat creating a PyTorch fixed seed tensor # Torch Seed torch . manual_seed ( 0 ) torch . rand ( 2 , 2 ) 0.5488 0.5928 0.7152 0.8443 [ torch . FloatTensor of size 2 x2 ] Creating a PyTorch tensor without seed Like with a numpy array of random numbers without seed, you will not get the same results as above. # Torch No Seed torch . rand ( 2 , 2 ) 0.6028 0.8579 0.5449 0.8473 [ torch . FloatTensor of size 2 x2 ] Repeat creating a PyTorch tensor without seed Notice how these are different numbers again? # Torch No Seed torch . rand ( 2 , 2 ) 0.4237 0.6236 0.6459 0.3844 [ torch . FloatTensor of size 2 x2 ] Seed for GPU is different for now... Fix a seed for GPU tensors When you conduct deep learning experiments, typically you want to use GPUs to accelerate your computations and fixing seed for tensors on GPUs is different from CPUs as we have done above. if torch . cuda . is_available (): torch . cuda . manual_seed_all ( 0 ) NumPy and Torch Bridge \u00b6 NumPy to Torch \u00b6 Create a numpy array of 1's # Numpy array np_array = np . ones (( 2 , 2 )) print ( np_array ) [[ 1. 1. ] [ 1. 1. ]] Get the type of class for the numpy array print ( type ( np_array )) < class ' numpy . ndarray '> Convert numpy array to PyTorch tensor # Convert to Torch Tensor torch_tensor = torch . from_numpy ( np_array ) print ( torch_tensor ) 1 1 1 1 [ torch . DoubleTensor of size 2 x2 ] Get type of class for PyTorch tensor Notice how it shows it's a torch DoubleTensor? There're actually tensor types and it depends on the numpy data type. print ( type ( torch_tensor )) < class ' torch . DoubleTensor '> Create PyTorch tensor from a different numpy datatype You will get an error running this code because PyTorch tensor don't support all datatype. # Data types matter: intentional error np_array_new = np . ones (( 2 , 2 ), dtype = np . int8 ) torch . from_numpy ( np_array_new ) --------------------------------------------------------------------------- RuntimeError Traceback ( most recent call last ) < ipython - input - 57 - b8b085f9b39d > in < module > () 1 # Data types matter 2 np_array_new = np . ones (( 2 , 2 ), dtype = np . int8 ) ----> 3 torch . from_numpy ( np_array_new ) RuntimeError : can 't convert a given np.ndarray to a tensor - it has an invalid type. The only supported types are: double, float, int64, int32, and uint8. What conversion support does Numpy to PyTorch tensor bridge gives? double float int64 , int32 , uint8 Create PyTorch long tensor See how a int64 numpy array gives you a PyTorch long tensor? # Data types matter np_array_new = np . ones (( 2 , 2 ), dtype = np . int64 ) torch . from_numpy ( np_array_new ) 1 1 1 1 [torch.LongTensor of size 2x2] Create PyTorch int tensor # Data types matter np_array_new = np . ones (( 2 , 2 ), dtype = np . int32 ) torch . from_numpy ( np_array_new ) 1 1 1 1 [ torch . IntTensor of size 2 x2 ] Create PyTorch byte tensor # Data types matter np_array_new = np . ones (( 2 , 2 ), dtype = np . uint8 ) torch . from_numpy ( np_array_new ) 1 1 1 1 [ torch . ByteTensor of size 2 x2 ] Create PyTorch Double Tensor # Data types matter np_array_new = np . ones (( 2 , 2 ), dtype = np . float64 ) torch . from_numpy ( np_array_new ) Alternatively you can do this too via np.double # Data types matter np_array_new = np . ones (( 2 , 2 ), dtype = np . double ) torch . from_numpy ( np_array_new ) 1 1 1 1 [ torch . DoubleTensor of size 2 x2 ] Create PyTorch Float Tensor # Data types matter np_array_new = np . ones (( 2 , 2 ), dtype = np . float32 ) torch . from_numpy ( np_array_new ) 1 1 1 1 [ torch . FloatTensor of size 2 x2 ] Summary Tensor Type Bug Guide These things don't matter much now. But later when you see error messages that require these particular tensor types, refer to this guide! NumPy Array Type Torch Tensor Type int64 LongTensor int32 IntegerTensor uint8 ByteTensor float64 DoubleTensor float32 FloatTensor double DoubleTensor Torch to NumPy \u00b6 Create PyTorch tensor of 1's You would realize this defaults to a float tensor by default if you do this. torch_tensor = torch . ones ( 2 , 2 ) type ( torch_tensor ) torch . FloatTensor Convert tensor to numpy It's as simple as this. torch_to_numpy = torch_tensor . numpy () type ( torch_to_numpy ) # Wowza, we did it. numpy . ndarray Tensors on CPU vs GPU \u00b6 Move tensor to CPU and back This by default creates a tensor on CPU. You do not need to do anything. # CPU tensor_cpu = torch . ones ( 2 , 2 ) If you would like to send a tensor to your GPU, you just need to do a simple .cuda() # CPU to GPU device = torch . device ( \"cuda:0\" if torch . cuda . is_available () else \"cpu\" ) tensor_cpu . to ( device ) And if you want to move that tensor on the GPU back to the CPU, just do the following. # GPU to CPU tensor_cpu . cpu () Tensor Operations \u00b6 Resizing Tensor \u00b6 Creating a 2x2 tensor a = torch . ones ( 2 , 2 ) print ( a ) 1 1 1 1 [ torch . FloatTensor of size 2 x2 ] Getting size of tensor print ( a . size ()) torch . Size ([ 2 , 2 ]) Resize tensor to 4x1 a . view ( 4 ) 1 1 1 1 [ torch . FloatTensor of size 4 ] Get size of resized tensor a . view ( 4 ) . size () torch . Size ([ 4 ]) Element-wise Addition \u00b6 Creating first 2x2 tensor a = torch . ones ( 2 , 2 ) print ( a ) 1 1 1 1 [ torch . FloatTensor of size 2 x2 ] Creating second 2x2 tensor b = torch . ones ( 2 , 2 ) print ( b ) 1 1 1 1 [ torch . FloatTensor of size 2 x2 ] Element-wise addition of 2 tensors # Element-wise addition c = a + b print ( c ) 2 2 2 2 [ torch . FloatTensor of size 2 x2 ] Alternative element-wise addition of 2 tensors # Element-wise addition c = torch . add ( a , b ) print ( c ) 2 2 2 2 [ torch . FloatTensor of size 2 x2 ] In-place element-wise addition This would replace the c tensor values with the new addition. # In-place addition print ( 'Old c tensor' ) print ( c ) c . add_ ( a ) print ( '-' * 60 ) print ( 'New c tensor' ) print ( c ) Old c tensor 2 2 2 2 [ torch . FloatTensor of size 2 x2 ] ------------------------------------------------------------ New c tensor 3 3 3 3 [ torch . FloatTensor of size 2 x2 ] Element-wise Subtraction \u00b6 Check values of tensor a and b' Take note that you've created tensor a and b of sizes 2x2 filled with 1's each above. print ( a ) print ( b ) 1 1 1 1 [ torch . FloatTensor of size 2 x2 ] 1 1 1 1 [ torch . FloatTensor of size 2 x2 ] Element-wise subtraction: method 1 a - b 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] Element-wise subtraction: method 2 # Not in-place print ( a . sub ( b )) print ( a ) 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] 1 1 1 1 [ torch . FloatTensor of size 2 x2 ] Element-wise subtraction: method 3 This will replace a with the final result filled with 2's # Inplace print ( a . sub_ ( b )) print ( a ) 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] Element-Wise Multiplication \u00b6 Create tensor a and b of sizes 2x2 filled with 1's and 0's a = torch . ones ( 2 , 2 ) print ( a ) b = torch . zeros ( 2 , 2 ) print ( b ) 1 1 1 1 [ torch . FloatTensor of size 2 x2 ] 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] Element-wise multiplication: method 1 a * b 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] Element-wise multiplication: method 2 # Not in-place print ( torch . mul ( a , b )) print ( a ) 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] 1 1 1 1 [ torch . FloatTensor of size 2 x2 ] Element-wise multiplication: method 3 # In-place print ( a . mul_ ( b )) print ( a ) 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] Element-Wise Division \u00b6 Create tensor a and b of sizes 2x2 filled with 1's and 0's a = torch . ones ( 2 , 2 ) print ( a ) b = torch . zeros ( 2 , 2 ) print ( b ) 1 1 1 1 [ torch . FloatTensor of size 2 x2 ] 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] Element-wise division: method 1 b / a 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] Element-wise division: method 2 torch . div ( b , a ) 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] Element-wise division: method 3 # Inplace b . div_ ( a ) 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] Tensor Mean \u00b6 \\[1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 = 55\\] \\[ mean = 55 /10 = 5.5 \\] Create tensor of size 10 filled from 1 to 10 a = torch . Tensor ([ 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 ]) a . size () torch . Size ([ 10 ]) Get tensor mean Here we get 5.5 as we've calculated manually above. a . mean ( dim = 0 ) 5.5000 [ torch . FloatTensor of size 1 ] Get tensor mean on second dimension Here we get an error because the tensor is of size 10 and not 10x1 so there's no second dimension to calculate. a . mean ( dim = 1 ) RuntimeError Traceback ( most recent call last ) < ipython - input - 7 - 81 aec0cf1c00 > in < module > () ----> 1 a . mean ( dim = 1 ) RuntimeError : dimension out of range ( expected to be in range of [ - 1 , 0 ], but got 1 ) Create a 2x10 Tensor, of 1-10 digits each a = torch . Tensor ([[ 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 ], [ 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 ]]) a . size () torch . Size ([ 2 , 10 ]) Get tensor mean on second dimension Here we won't get an error like previously because we've a tensor of size 2x10 a . mean ( dim = 1 ) 5.5000 5.5000 [ torch . FloatTensor of size 2 x1 ] Tensor Standard Deviation \u00b6 Get standard deviation of tensor a = torch . Tensor ([ 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 ]) a . std ( dim = 0 ) 3.0277 [ torch . FloatTensor of size 1 ] Summary \u00b6 We've learnt to... Success Create Matrices Create Matrices with Default Initialization Values Zeros Ones Initialize Seeds for Reproducibility on GPU and CPU Convert Matrices: NumPy to Torch and Torch to NumPy Move Tensors: CPU to GPU and GPU to CPU Run Important Tensor Operations Element-wise addition, subtraction, multiplication and division Resize Calculate mean Calculate standard deviation Citation \u00b6 If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Matrices"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#matrices-with-pytorch","text":"Run Jupyter Notebook You can run the code for this section in this jupyter notebook link .","title":"Matrices with PyTorch"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#matrices","text":"","title":"Matrices"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#matrices-brief-introduction","text":"Basic definition: rectangular array of numbers. Tensors (PyTorch) Ndarrays (NumPy) 2 x 2 Matrix (R x C) 1 1 1 1 2 x 3 Matrix 1 1 1 1 1 1","title":"Matrices Brief Introduction"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#creating-matrices","text":"Create list # Creating a 2x2 array arr = [[ 1 , 2 ], [ 3 , 4 ]] print ( arr ) [[ 1 , 2 ], [ 3 , 4 ]] Create numpy array via list import numpy as np # Convert to NumPy np . array ( arr ) array ([[ 1 , 2 ], [ 3 , 4 ]]) Convert numpy array to PyTorch tensor import torch # Convert to PyTorch Tensor torch . Tensor ( arr ) 1 2 3 4 [ torch . FloatTensor of size 2 x2 ]","title":"Creating Matrices"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#create-matrices-with-default-values","text":"Create 2x2 numpy array of 1's np . ones (( 2 , 2 )) array ([[ 1. , 1. ], [ 1. , 1. ]]) Create 2x2 torch tensor of 1's torch . ones (( 2 , 2 )) 1 1 1 1 [torch.FloatTensor of size 2x2] Create 2x2 numpy array of random numbers np . random . rand ( 2 , 2 ) array ([[ 0.68270631 , 0.87721678 ], [ 0.07420986 , 0.79669375 ]]) Create 2x2 PyTorch tensor of random numbers torch . rand ( 2 , 2 ) 0.3900 0.8268 0.3888 0.5914 [ torch . FloatTensor of size 2 x2 ]","title":"Create Matrices with Default Values"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#seeds-for-reproducibility","text":"Why do we need seeds? We need seeds to enable reproduction of experimental results. This becomes critical later on where you can easily let people reproduce your code's output exactly as you've produced. Create seed to enable fixed numbers for random number generation # Seed np . random . seed ( 0 ) np . random . rand ( 2 , 2 ) array ([[ 0.5488135 , 0.71518937 ], [ 0.60276338 , 0.54488318 ]]) Repeat random array generation to check If you do not set the seed, you would not get the same set of numbers like here. # Seed np . random . seed ( 0 ) np . random . rand ( 2 , 2 ) array ([[ 0.5488135 , 0.71518937 ], [ 0.60276338 , 0.54488318 ]]) Create a numpy array without seed Notice how you get different numbers compared to the first 2 tries? # No seed np . random . rand ( 2 , 2 ) array ([[ 0.56804456 , 0.92559664 ], [ 0.07103606 , 0.0871293 ]]) Repeat numpy array generation without seed You get the point now, you get a totally different set of numbers. # No seed np . random . rand ( 2 , 2 ) array ([[ 0.0202184 , 0.83261985 ], [ 0.77815675 , 0.87001215 ]]) Create a PyTorch tensor with a fixed seed # Torch Seed torch . manual_seed ( 0 ) torch . rand ( 2 , 2 ) 0.5488 0.5928 0.7152 0.8443 [ torch . FloatTensor of size 2 x2 ] Repeat creating a PyTorch fixed seed tensor # Torch Seed torch . manual_seed ( 0 ) torch . rand ( 2 , 2 ) 0.5488 0.5928 0.7152 0.8443 [ torch . FloatTensor of size 2 x2 ] Creating a PyTorch tensor without seed Like with a numpy array of random numbers without seed, you will not get the same results as above. # Torch No Seed torch . rand ( 2 , 2 ) 0.6028 0.8579 0.5449 0.8473 [ torch . FloatTensor of size 2 x2 ] Repeat creating a PyTorch tensor without seed Notice how these are different numbers again? # Torch No Seed torch . rand ( 2 , 2 ) 0.4237 0.6236 0.6459 0.3844 [ torch . FloatTensor of size 2 x2 ] Seed for GPU is different for now... Fix a seed for GPU tensors When you conduct deep learning experiments, typically you want to use GPUs to accelerate your computations and fixing seed for tensors on GPUs is different from CPUs as we have done above. if torch . cuda . is_available (): torch . cuda . manual_seed_all ( 0 )","title":"Seeds for Reproducibility"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#numpy-and-torch-bridge","text":"","title":"NumPy and Torch Bridge"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#numpy-to-torch","text":"Create a numpy array of 1's # Numpy array np_array = np . ones (( 2 , 2 )) print ( np_array ) [[ 1. 1. ] [ 1. 1. ]] Get the type of class for the numpy array print ( type ( np_array )) < class ' numpy . ndarray '> Convert numpy array to PyTorch tensor # Convert to Torch Tensor torch_tensor = torch . from_numpy ( np_array ) print ( torch_tensor ) 1 1 1 1 [ torch . DoubleTensor of size 2 x2 ] Get type of class for PyTorch tensor Notice how it shows it's a torch DoubleTensor? There're actually tensor types and it depends on the numpy data type. print ( type ( torch_tensor )) < class ' torch . DoubleTensor '> Create PyTorch tensor from a different numpy datatype You will get an error running this code because PyTorch tensor don't support all datatype. # Data types matter: intentional error np_array_new = np . ones (( 2 , 2 ), dtype = np . int8 ) torch . from_numpy ( np_array_new ) --------------------------------------------------------------------------- RuntimeError Traceback ( most recent call last ) < ipython - input - 57 - b8b085f9b39d > in < module > () 1 # Data types matter 2 np_array_new = np . ones (( 2 , 2 ), dtype = np . int8 ) ----> 3 torch . from_numpy ( np_array_new ) RuntimeError : can 't convert a given np.ndarray to a tensor - it has an invalid type. The only supported types are: double, float, int64, int32, and uint8. What conversion support does Numpy to PyTorch tensor bridge gives? double float int64 , int32 , uint8 Create PyTorch long tensor See how a int64 numpy array gives you a PyTorch long tensor? # Data types matter np_array_new = np . ones (( 2 , 2 ), dtype = np . int64 ) torch . from_numpy ( np_array_new ) 1 1 1 1 [torch.LongTensor of size 2x2] Create PyTorch int tensor # Data types matter np_array_new = np . ones (( 2 , 2 ), dtype = np . int32 ) torch . from_numpy ( np_array_new ) 1 1 1 1 [ torch . IntTensor of size 2 x2 ] Create PyTorch byte tensor # Data types matter np_array_new = np . ones (( 2 , 2 ), dtype = np . uint8 ) torch . from_numpy ( np_array_new ) 1 1 1 1 [ torch . ByteTensor of size 2 x2 ] Create PyTorch Double Tensor # Data types matter np_array_new = np . ones (( 2 , 2 ), dtype = np . float64 ) torch . from_numpy ( np_array_new ) Alternatively you can do this too via np.double # Data types matter np_array_new = np . ones (( 2 , 2 ), dtype = np . double ) torch . from_numpy ( np_array_new ) 1 1 1 1 [ torch . DoubleTensor of size 2 x2 ] Create PyTorch Float Tensor # Data types matter np_array_new = np . ones (( 2 , 2 ), dtype = np . float32 ) torch . from_numpy ( np_array_new ) 1 1 1 1 [ torch . FloatTensor of size 2 x2 ] Summary Tensor Type Bug Guide These things don't matter much now. But later when you see error messages that require these particular tensor types, refer to this guide! NumPy Array Type Torch Tensor Type int64 LongTensor int32 IntegerTensor uint8 ByteTensor float64 DoubleTensor float32 FloatTensor double DoubleTensor","title":"NumPy to Torch"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#torch-to-numpy","text":"Create PyTorch tensor of 1's You would realize this defaults to a float tensor by default if you do this. torch_tensor = torch . ones ( 2 , 2 ) type ( torch_tensor ) torch . FloatTensor Convert tensor to numpy It's as simple as this. torch_to_numpy = torch_tensor . numpy () type ( torch_to_numpy ) # Wowza, we did it. numpy . ndarray","title":"Torch to NumPy"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#tensors-on-cpu-vs-gpu","text":"Move tensor to CPU and back This by default creates a tensor on CPU. You do not need to do anything. # CPU tensor_cpu = torch . ones ( 2 , 2 ) If you would like to send a tensor to your GPU, you just need to do a simple .cuda() # CPU to GPU device = torch . device ( \"cuda:0\" if torch . cuda . is_available () else \"cpu\" ) tensor_cpu . to ( device ) And if you want to move that tensor on the GPU back to the CPU, just do the following. # GPU to CPU tensor_cpu . cpu ()","title":"Tensors on CPU vs GPU"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#tensor-operations","text":"","title":"Tensor Operations"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#resizing-tensor","text":"Creating a 2x2 tensor a = torch . ones ( 2 , 2 ) print ( a ) 1 1 1 1 [ torch . FloatTensor of size 2 x2 ] Getting size of tensor print ( a . size ()) torch . Size ([ 2 , 2 ]) Resize tensor to 4x1 a . view ( 4 ) 1 1 1 1 [ torch . FloatTensor of size 4 ] Get size of resized tensor a . view ( 4 ) . size () torch . Size ([ 4 ])","title":"Resizing Tensor"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#element-wise-addition","text":"Creating first 2x2 tensor a = torch . ones ( 2 , 2 ) print ( a ) 1 1 1 1 [ torch . FloatTensor of size 2 x2 ] Creating second 2x2 tensor b = torch . ones ( 2 , 2 ) print ( b ) 1 1 1 1 [ torch . FloatTensor of size 2 x2 ] Element-wise addition of 2 tensors # Element-wise addition c = a + b print ( c ) 2 2 2 2 [ torch . FloatTensor of size 2 x2 ] Alternative element-wise addition of 2 tensors # Element-wise addition c = torch . add ( a , b ) print ( c ) 2 2 2 2 [ torch . FloatTensor of size 2 x2 ] In-place element-wise addition This would replace the c tensor values with the new addition. # In-place addition print ( 'Old c tensor' ) print ( c ) c . add_ ( a ) print ( '-' * 60 ) print ( 'New c tensor' ) print ( c ) Old c tensor 2 2 2 2 [ torch . FloatTensor of size 2 x2 ] ------------------------------------------------------------ New c tensor 3 3 3 3 [ torch . FloatTensor of size 2 x2 ]","title":"Element-wise Addition"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#element-wise-subtraction","text":"Check values of tensor a and b' Take note that you've created tensor a and b of sizes 2x2 filled with 1's each above. print ( a ) print ( b ) 1 1 1 1 [ torch . FloatTensor of size 2 x2 ] 1 1 1 1 [ torch . FloatTensor of size 2 x2 ] Element-wise subtraction: method 1 a - b 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] Element-wise subtraction: method 2 # Not in-place print ( a . sub ( b )) print ( a ) 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] 1 1 1 1 [ torch . FloatTensor of size 2 x2 ] Element-wise subtraction: method 3 This will replace a with the final result filled with 2's # Inplace print ( a . sub_ ( b )) print ( a ) 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] 0 0 0 0 [ torch . FloatTensor of size 2 x2 ]","title":"Element-wise Subtraction"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#element-wise-multiplication","text":"Create tensor a and b of sizes 2x2 filled with 1's and 0's a = torch . ones ( 2 , 2 ) print ( a ) b = torch . zeros ( 2 , 2 ) print ( b ) 1 1 1 1 [ torch . FloatTensor of size 2 x2 ] 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] Element-wise multiplication: method 1 a * b 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] Element-wise multiplication: method 2 # Not in-place print ( torch . mul ( a , b )) print ( a ) 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] 1 1 1 1 [ torch . FloatTensor of size 2 x2 ] Element-wise multiplication: method 3 # In-place print ( a . mul_ ( b )) print ( a ) 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] 0 0 0 0 [ torch . FloatTensor of size 2 x2 ]","title":"Element-Wise Multiplication"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#element-wise-division","text":"Create tensor a and b of sizes 2x2 filled with 1's and 0's a = torch . ones ( 2 , 2 ) print ( a ) b = torch . zeros ( 2 , 2 ) print ( b ) 1 1 1 1 [ torch . FloatTensor of size 2 x2 ] 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] Element-wise division: method 1 b / a 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] Element-wise division: method 2 torch . div ( b , a ) 0 0 0 0 [ torch . FloatTensor of size 2 x2 ] Element-wise division: method 3 # Inplace b . div_ ( a ) 0 0 0 0 [ torch . FloatTensor of size 2 x2 ]","title":"Element-Wise Division"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#tensor-mean","text":"\\[1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 = 55\\] \\[ mean = 55 /10 = 5.5 \\] Create tensor of size 10 filled from 1 to 10 a = torch . Tensor ([ 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 ]) a . size () torch . Size ([ 10 ]) Get tensor mean Here we get 5.5 as we've calculated manually above. a . mean ( dim = 0 ) 5.5000 [ torch . FloatTensor of size 1 ] Get tensor mean on second dimension Here we get an error because the tensor is of size 10 and not 10x1 so there's no second dimension to calculate. a . mean ( dim = 1 ) RuntimeError Traceback ( most recent call last ) < ipython - input - 7 - 81 aec0cf1c00 > in < module > () ----> 1 a . mean ( dim = 1 ) RuntimeError : dimension out of range ( expected to be in range of [ - 1 , 0 ], but got 1 ) Create a 2x10 Tensor, of 1-10 digits each a = torch . Tensor ([[ 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 ], [ 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 ]]) a . size () torch . Size ([ 2 , 10 ]) Get tensor mean on second dimension Here we won't get an error like previously because we've a tensor of size 2x10 a . mean ( dim = 1 ) 5.5000 5.5000 [ torch . FloatTensor of size 2 x1 ]","title":"Tensor Mean"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#tensor-standard-deviation","text":"Get standard deviation of tensor a = torch . Tensor ([ 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 ]) a . std ( dim = 0 ) 3.0277 [ torch . FloatTensor of size 1 ]","title":"Tensor Standard Deviation"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#summary","text":"We've learnt to... Success Create Matrices Create Matrices with Default Initialization Values Zeros Ones Initialize Seeds for Reproducibility on GPU and CPU Convert Matrices: NumPy to Torch and Torch to NumPy Move Tensors: CPU to GPU and GPU to CPU Run Important Tensor Operations Element-wise addition, subtraction, multiplication and division Resize Calculate mean Calculate standard deviation","title":"Summary"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#citation","text":"If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Citation"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/","text":"Recurrent Neural Network with PyTorch \u00b6 Run Jupyter Notebook You can run the code for this section in this jupyter notebook link . About Recurrent Neural Network \u00b6 Feedforward Neural Networks Transition to 1 Layer Recurrent Neural Networks (RNN) \u00b6 RNN is essentially an FNN but with a hidden layer (non-linear output) that passes on information to the next FNN Compared to an FNN, we've one additional set of weight and bias that allows information to flow from one FNN to another FNN sequentially that allows time-dependency. The diagram below shows the only difference between an FNN and a RNN. 2 Layer RNN Breakdown \u00b6 Building a Recurrent Neural Network with PyTorch \u00b6 Model A: 1 Hidden Layer (ReLU) \u00b6 Unroll 28 time steps Each step input size: 28 x 1 Total per unroll: 28 x 28 Feedforward Neural Network input size: 28 x 28 1 Hidden layer ReLU Activation Function Steps \u00b6 Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model Step 1: Loading MNIST Train Dataset \u00b6 Images from 1 to 9 Looking into the MNIST Dataset import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) We would have 60k training images of size 28 x 28 pixels. print ( train_dataset . train_data . size ()) print ( train_dataset . train_labels . size ()) Here we would have 10k testing images of the same size, 28 x 28 pixels. print ( test_dataset . test_data . size ()) print ( test_dataset . test_labels . size ()) torch . Size ([ 60000 , 28 , 28 ]) torch . Size ([ 60000 ]) torch . Size ([ 10000 , 28 , 28 ]) torch . Size ([ 10000 ]) Step 2: Make Dataset Iterable \u00b6 Creating iterable objects to loop through subsequently batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) Step 3: Create Model Class \u00b6 1 Layer RNN class RNNModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , layer_dim , output_dim ): super ( RNNModel , self ) . __init__ () # Hidden dimensions self . hidden_dim = hidden_dim # Number of hidden layers self . layer_dim = layer_dim # Building your RNN # batch_first=True causes input/output tensors to be of shape # (batch_dim, seq_dim, input_dim) # batch_dim = number of samples per batch self . rnn = nn . RNN ( input_dim , hidden_dim , layer_dim , batch_first = True , nonlinearity = 'relu' ) # Readout layer self . fc = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Initialize hidden state with zeros # (layer_dim, batch_size, hidden_dim) h0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () # We need to detach the hidden state to prevent exploding/vanishing gradients # This is part of truncated backpropagation through time (BPTT) out , hn = self . rnn ( x , h0 . detach ()) # Index hidden state of last time step # out.size() --> 100, 28, 10 # out[:, -1, :] --> 100, 10 --> just want last time step hidden states! out = self . fc ( out [:, - 1 , :]) # out.size() --> 100, 10 return out Step 4: Instantiate Model Class \u00b6 28 time steps Each time step: input dimension = 28 1 hidden layer MNIST 1-9 digits \\(\\rightarrow\\) output dimension = 10 Instantiate model class and assign to an object input_dim = 28 hidden_dim = 100 layer_dim = 1 output_dim = 10 model = RNNModel ( input_dim , hidden_dim , layer_dim , output_dim ) Step 5: Instantiate Loss Class \u00b6 Recurrent Neural Network: Cross Entropy Loss Convolutional Neural Network : Cross Entropy Loss Feedforward Neural Network : Cross Entropy Loss Logistic Regression : Cross Entropy Loss Linear Regression : MSE Cross Entropy Loss for Classification Task criterion = nn . CrossEntropyLoss () Cross Entropy vs MSE Take note that there are cases where RNN, CNN and FNN use MSE as a loss function. We use cross entropy for classification tasks (predicting 0-9 digits in MNIST for example). And we use MSE for regression tasks (predicting temperatures in every December in San Francisco for example). Step 6: Instantiate Optimizer Class \u00b6 Simplified equation \\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\) \\(\\theta\\) : parameters (our tensors with gradient accumulation abilities) \\(\\eta\\) : learning rate (how fast we want to learn) \\(\\nabla_\\theta\\) : gradients of loss with respect to the model's parameters Even simplier equation parameters = parameters - learning_rate * parameters_gradients At every iteration, we update our model's parameters learning_rate = 0.01 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) Parameters In-Depth \u00b6 Input to Hidden Layer Affine Function A1, B1 Hidden Layer to Output Affine Function A2, B2 Hidden Layer to Hidden Layer Affine Function A3, B3 Total groups of parameters We should have 6 groups as shown above. len ( list ( model . parameters ())) 6 Input to Hidden Weight Remember we defined our hidden layer to have a size of 100. Because our input is a size of 28 at each time step, this gives rise to a weight matrix of 100 x 28. # Input --> Hidden (A1) list ( model . parameters ())[ 0 ] . size () torch . Size ([ 100 , 28 ]) Input to Hidden Bias # Input --> Hidden BIAS (B1) list ( model . parameters ())[ 2 ] . size () torch . Size ([ 100 ]) Hidden to Hidden # Hidden --> Hidden (A3) list ( model . parameters ())[ 1 ] . size () torch . Size ([ 100 , 100 ]) Hidden to Hidden Bias # Hidden --> Hidden BIAS(B3) list ( model . parameters ())[ 3 ] . size () torch . Size ([ 100 ]) Hidden to Output # Hidden --> Output (A2) list ( model . parameters ())[ 4 ] . size () torch . Size ([ 10 , 100 ]) Hidden to Output Bias # Hidden --> Output BIAS (B2) list ( model . parameters ())[ 5 ] . size () torch . Size ([ 10 ]) Step 7: Train Model \u00b6 Process Convert inputs/labels to tensors with gradient accumulation abilities RNN Input: (1, 28) CNN Input: (1, 28, 28) FNN Input: (1, 28*28) Clear gradient buffets Get output given inputs Get loss Get gradients w.r.t. parameters Update parameters using gradients parameters = parameters - learning_rate * parameters_gradients REPEAT Same 7 step process for training models # Number of steps to unroll seq_dim = 28 iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): model . train () # Load images as tensors with gradient accumulation abilities images = images . view ( - 1 , seq_dim , input_dim ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits # outputs.size() --> 100, 10 outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : model . eval () # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch tensors with gradient accumulation abilities images = images . view ( - 1 , seq_dim , input_dim ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 2.301494836807251 . Accuracy : 12 Iteration : 1000. Loss : 2.2986037731170654 . Accuracy : 14 Iteration : 1500. Loss : 2.278566598892212 . Accuracy : 18 Iteration : 2000. Loss : 2.169614315032959 . Accuracy : 21 Iteration : 2500. Loss : 1.1662731170654297 . Accuracy : 51 Iteration : 3000. Loss : 0.9290509223937988 . Accuracy : 71 Model B: 2 Hidden Layer (ReLU) \u00b6 Unroll 28 time steps Each step input size: 28 x 1 Total per unroll: 28 x 28 Feedforward Neural Network inpt size: 28 x 28 2 Hidden layer ReLU Activation Function Steps \u00b6 Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 2 Hidden Layer + ReLU import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class RNNModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , layer_dim , output_dim ): super ( RNNModel , self ) . __init__ () # Hidden dimensions self . hidden_dim = hidden_dim # Number of hidden layers self . layer_dim = layer_dim # Building your RNN # batch_first=True causes input/output tensors to be of shape # (batch_dim, seq_dim, feature_dim) self . rnn = nn . RNN ( input_dim , hidden_dim , layer_dim , batch_first = True , nonlinearity = 'relu' ) # Readout layer self . fc = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Initialize hidden state with zeros h0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () # We need to detach the hidden state to prevent exploding/vanishing gradients # This is part of truncated backpropagation through time (BPTT) out , hn = self . rnn ( x , h0 . detach ()) # Index hidden state of last time step # out.size() --> 100, 28, 100 # out[:, -1, :] --> 100, 100 --> just want last time step hidden states! out = self . fc ( out [:, - 1 , :]) # out.size() --> 100, 10 return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 hidden_dim = 100 layer_dim = 2 # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER output_dim = 10 model = RNNModel ( input_dim , hidden_dim , layer_dim , output_dim ) # JUST PRINTING MODEL & PARAMETERS print ( model ) print ( len ( list ( model . parameters ()))) for i in range ( len ( list ( model . parameters ()))): print ( list ( model . parameters ())[ i ] . size ()) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.01 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' # Number of steps to unroll seq_dim = 28 iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): model . train () # Load images as tensors with gradient accumulation abilities images = images . view ( - 1 , seq_dim , input_dim ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits # outputs.size() --> 100, 10 outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : model . eval () # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Resize images images = images . view ( - 1 , seq_dim , input_dim ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) RNNModel ( ( rnn ): RNN ( 28 , 100 , num_layers = 2 , batch_first = True ) ( fc ): Linear ( in_features = 100 , out_features = 10 , bias = True ) ) 10 torch . Size ([ 100 , 28 ]) torch . Size ([ 100 , 100 ]) torch . Size ([ 100 ]) torch . Size ([ 100 ]) torch . Size ([ 100 , 100 ]) torch . Size ([ 100 , 100 ]) torch . Size ([ 100 ]) torch . Size ([ 100 ]) torch . Size ([ 10 , 100 ]) torch . Size ([ 10 ]) Iteration : 500. Loss : 2.3019518852233887 . Accuracy : 11 Iteration : 1000. Loss : 2.299217700958252 . Accuracy : 11 Iteration : 1500. Loss : 2.279090166091919 . Accuracy : 14 Iteration : 2000. Loss : 2.126953125 . Accuracy : 25 Iteration : 2500. Loss : 1.356347680091858 . Accuracy : 57 Iteration : 3000. Loss : 0.7377720475196838 . Accuracy : 69 10 sets of parameters First hidden Layer \\(A_1 = [100, 28]\\) \\(A_3 = [100, 100]\\) \\(B_1 = [100]\\) \\(B_3 = [100]\\) Second hidden layer \\(A_2 = [100, 100]\\) \\(A_5 = [100, 100]\\) \\(B_2 = [100]\\) \\(B_5 = [100]\\) Readout layer \\(A_4 = [10, 100]\\) \\(B_4 = [10]\\) Model C: 2 Hidden Layer \u00b6 Unroll 28 time steps Each step input size: 28 x 1 Total per unroll: 28 x 28 Feedforward Neural Network inpt size: 28 x 28 2 Hidden layer Tanh Activation Function Steps \u00b6 Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model !!! \"2 Hidden + ReLU\" import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class RNNModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , layer_dim , output_dim ): super ( RNNModel , self ) . __init__ () # Hidden dimensions self . hidden_dim = hidden_dim # Number of hidden layers self . layer_dim = layer_dim # Building your RNN # batch_first=True causes input/output tensors to be of shape # (batch_dim, seq_dim, feature_dim) self . rnn = nn . RNN ( input_dim , hidden_dim , layer_dim , batch_first = True , nonlinearity = 'tanh' ) # Readout layer self . fc = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Initialize hidden state with zeros h0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () # One time step # We need to detach the hidden state to prevent exploding/vanishing gradients # This is part of truncated backpropagation through time (BPTT) out , hn = self . rnn ( x , h0 . detach ()) # Index hidden state of last time step # out.size() --> 100, 28, 100 # out[:, -1, :] --> 100, 100 --> just want last time step hidden states! out = self . fc ( out [:, - 1 , :]) # out.size() --> 100, 10 return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 hidden_dim = 100 layer_dim = 2 # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER output_dim = 10 model = RNNModel ( input_dim , hidden_dim , layer_dim , output_dim ) # JUST PRINTING MODEL & PARAMETERS print ( model ) print ( len ( list ( model . parameters ()))) for i in range ( len ( list ( model . parameters ()))): print ( list ( model . parameters ())[ i ] . size ()) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' # Number of steps to unroll seq_dim = 28 iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as tensors with gradient accumulation abilities images = images . view ( - 1 , seq_dim , input_dim ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits # outputs.size() --> 100, 10 outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Resize images images = images . view ( - 1 , seq_dim , input_dim ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) RNNModel ( ( rnn ): RNN ( 28 , 100 , num_layers = 2 , batch_first = True ) ( fc ): Linear ( in_features = 100 , out_features = 10 , bias = True ) ) 10 torch . Size ([ 100 , 28 ]) torch . Size ([ 100 , 100 ]) torch . Size ([ 100 ]) torch . Size ([ 100 ]) torch . Size ([ 100 , 100 ]) torch . Size ([ 100 , 100 ]) torch . Size ([ 100 ]) torch . Size ([ 100 ]) torch . Size ([ 10 , 100 ]) torch . Size ([ 10 ]) Iteration : 500. Loss : 0.5943437218666077 . Accuracy : 77 Iteration : 1000. Loss : 0.22048641741275787 . Accuracy : 91 Iteration : 1500. Loss : 0.18479223549365997 . Accuracy : 94 Iteration : 2000. Loss : 0.2723771929740906 . Accuracy : 91 Iteration : 2500. Loss : 0.18817797303199768 . Accuracy : 92 Iteration : 3000. Loss : 0.1685929149389267 . Accuracy : 92 Summary of Results \u00b6 Model A Model B Model C ReLU ReLU Tanh 1 Hidden Layer 2 Hidden Layers 2 Hidden Layers 100 Hidden Units 100 Hidden Units 100 Hidden Units 92.48% 95.09% 95.54% General Deep Learning Notes \u00b6 2 ways to expand a recurrent neural network More non-linear activation units (neurons) More hidden layers Cons Need a larger dataset Curse of dimensionality Does not necessarily mean higher accuracy 3. Building a Recurrent Neural Network with PyTorch (GPU) \u00b6 Model C: 2 Hidden Layer (Tanh) \u00b6 GPU: 2 things must be on GPU - model - tensors Steps \u00b6 Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 2 Layer RNN + Tanh import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class RNNModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , layer_dim , output_dim ): super ( RNNModel , self ) . __init__ () # Hidden dimensions self . hidden_dim = hidden_dim # Number of hidden layers self . layer_dim = layer_dim # Building your RNN # batch_first=True causes input/output tensors to be of shape # (batch_dim, seq_dim, feature_dim) self . rnn = nn . RNN ( input_dim , hidden_dim , layer_dim , batch_first = True , nonlinearity = 'tanh' ) # Readout layer self . fc = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Initialize hidden state with zeros ####################### # USE GPU FOR MODEL # ####################### h0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . to ( device ) # One time step # We need to detach the hidden state to prevent exploding/vanishing gradients # This is part of truncated backpropagation through time (BPTT) out , hn = self . rnn ( x , h0 . detach ()) # Index hidden state of last time step # out.size() --> 100, 28, 100 # out[:, -1, :] --> 100, 100 --> just want last time step hidden states! out = self . fc ( out [:, - 1 , :]) # out.size() --> 100, 10 return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 hidden_dim = 100 layer_dim = 2 # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER output_dim = 10 model = RNNModel ( input_dim , hidden_dim , layer_dim , output_dim ) ####################### # USE GPU FOR MODEL # ####################### device = torch . device ( \"cuda:0\" if torch . cuda . is_available () else \"cpu\" ) model . to ( device ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' # Number of steps to unroll seq_dim = 28 iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as tensors with gradient accumulation abilities ####################### # USE GPU FOR MODEL # ####################### images = images . view ( - 1 , seq_dim , input_dim ) . requires_grad_ () . to ( device ) labels = labels . to ( device ) # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits # outputs.size() --> 100, 10 outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : ####################### # USE GPU FOR MODEL # ####################### images = images . view ( - 1 , seq_dim , input_dim ) . to ( device ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions ####################### # USE GPU FOR MODEL # ####################### if torch . cuda . is_available (): correct += ( predicted . cpu () == labels . cpu ()) . sum () else : correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 0.5983774662017822 . Accuracy : 81 Iteration : 1000. Loss : 0.2960105836391449 . Accuracy : 86 Iteration : 1500. Loss : 0.19428101181983948 . Accuracy : 93 Iteration : 2000. Loss : 0.11918395012617111 . Accuracy : 95 Iteration : 2500. Loss : 0.11246936023235321 . Accuracy : 95 Iteration : 3000. Loss : 0.15849310159683228 . Accuracy : 95 Summary \u00b6 We've learnt to... Success Feedforward Neural Networks Transition to Recurrent Neural Networks RNN Models in PyTorch Model A: 1 Hidden Layer RNN (ReLU) Model B: 2 Hidden Layer RNN (ReLU) Model C: 2 Hidden Layer RNN (Tanh) Models Variation in Code Modifying only step 4 Ways to Expand Model\u2019s Capacity More non-linear activation units ( neurons ) More hidden layers Cons of Expanding Capacity Need more data Does not necessarily mean higher accuracy GPU Code 2 things on GPU model tensors with gradient accumulation abilities Modifying only Step 3, 4 and 7 7 Step Model Building Recap Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model Step 7: Train Model Citation \u00b6 If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Recurrent Neural Networks (RNN)"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#recurrent-neural-network-with-pytorch","text":"Run Jupyter Notebook You can run the code for this section in this jupyter notebook link .","title":"Recurrent Neural Network with PyTorch"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#about-recurrent-neural-network","text":"","title":"About Recurrent Neural Network"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#feedforward-neural-networks-transition-to-1-layer-recurrent-neural-networks-rnn","text":"RNN is essentially an FNN but with a hidden layer (non-linear output) that passes on information to the next FNN Compared to an FNN, we've one additional set of weight and bias that allows information to flow from one FNN to another FNN sequentially that allows time-dependency. The diagram below shows the only difference between an FNN and a RNN.","title":"Feedforward Neural Networks Transition to 1 Layer Recurrent Neural Networks (RNN)"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#2-layer-rnn-breakdown","text":"","title":"2 Layer RNN Breakdown"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#building-a-recurrent-neural-network-with-pytorch","text":"","title":"Building a Recurrent Neural Network with PyTorch"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#model-a-1-hidden-layer-relu","text":"Unroll 28 time steps Each step input size: 28 x 1 Total per unroll: 28 x 28 Feedforward Neural Network input size: 28 x 28 1 Hidden layer ReLU Activation Function","title":"Model A: 1 Hidden Layer (ReLU)"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#steps","text":"Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model","title":"Steps"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#step-1-loading-mnist-train-dataset","text":"Images from 1 to 9 Looking into the MNIST Dataset import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) We would have 60k training images of size 28 x 28 pixels. print ( train_dataset . train_data . size ()) print ( train_dataset . train_labels . size ()) Here we would have 10k testing images of the same size, 28 x 28 pixels. print ( test_dataset . test_data . size ()) print ( test_dataset . test_labels . size ()) torch . Size ([ 60000 , 28 , 28 ]) torch . Size ([ 60000 ]) torch . Size ([ 10000 , 28 , 28 ]) torch . Size ([ 10000 ])","title":"Step 1: Loading MNIST Train Dataset"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#step-2-make-dataset-iterable","text":"Creating iterable objects to loop through subsequently batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False )","title":"Step 2: Make Dataset Iterable"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#step-3-create-model-class","text":"1 Layer RNN class RNNModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , layer_dim , output_dim ): super ( RNNModel , self ) . __init__ () # Hidden dimensions self . hidden_dim = hidden_dim # Number of hidden layers self . layer_dim = layer_dim # Building your RNN # batch_first=True causes input/output tensors to be of shape # (batch_dim, seq_dim, input_dim) # batch_dim = number of samples per batch self . rnn = nn . RNN ( input_dim , hidden_dim , layer_dim , batch_first = True , nonlinearity = 'relu' ) # Readout layer self . fc = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Initialize hidden state with zeros # (layer_dim, batch_size, hidden_dim) h0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () # We need to detach the hidden state to prevent exploding/vanishing gradients # This is part of truncated backpropagation through time (BPTT) out , hn = self . rnn ( x , h0 . detach ()) # Index hidden state of last time step # out.size() --> 100, 28, 10 # out[:, -1, :] --> 100, 10 --> just want last time step hidden states! out = self . fc ( out [:, - 1 , :]) # out.size() --> 100, 10 return out","title":"Step 3: Create Model Class"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#step-4-instantiate-model-class","text":"28 time steps Each time step: input dimension = 28 1 hidden layer MNIST 1-9 digits \\(\\rightarrow\\) output dimension = 10 Instantiate model class and assign to an object input_dim = 28 hidden_dim = 100 layer_dim = 1 output_dim = 10 model = RNNModel ( input_dim , hidden_dim , layer_dim , output_dim )","title":"Step 4: Instantiate Model Class"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#step-5-instantiate-loss-class","text":"Recurrent Neural Network: Cross Entropy Loss Convolutional Neural Network : Cross Entropy Loss Feedforward Neural Network : Cross Entropy Loss Logistic Regression : Cross Entropy Loss Linear Regression : MSE Cross Entropy Loss for Classification Task criterion = nn . CrossEntropyLoss () Cross Entropy vs MSE Take note that there are cases where RNN, CNN and FNN use MSE as a loss function. We use cross entropy for classification tasks (predicting 0-9 digits in MNIST for example). And we use MSE for regression tasks (predicting temperatures in every December in San Francisco for example).","title":"Step 5: Instantiate Loss Class"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#step-6-instantiate-optimizer-class","text":"Simplified equation \\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\) \\(\\theta\\) : parameters (our tensors with gradient accumulation abilities) \\(\\eta\\) : learning rate (how fast we want to learn) \\(\\nabla_\\theta\\) : gradients of loss with respect to the model's parameters Even simplier equation parameters = parameters - learning_rate * parameters_gradients At every iteration, we update our model's parameters learning_rate = 0.01 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate )","title":"Step 6: Instantiate Optimizer Class"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#parameters-in-depth","text":"Input to Hidden Layer Affine Function A1, B1 Hidden Layer to Output Affine Function A2, B2 Hidden Layer to Hidden Layer Affine Function A3, B3 Total groups of parameters We should have 6 groups as shown above. len ( list ( model . parameters ())) 6 Input to Hidden Weight Remember we defined our hidden layer to have a size of 100. Because our input is a size of 28 at each time step, this gives rise to a weight matrix of 100 x 28. # Input --> Hidden (A1) list ( model . parameters ())[ 0 ] . size () torch . Size ([ 100 , 28 ]) Input to Hidden Bias # Input --> Hidden BIAS (B1) list ( model . parameters ())[ 2 ] . size () torch . Size ([ 100 ]) Hidden to Hidden # Hidden --> Hidden (A3) list ( model . parameters ())[ 1 ] . size () torch . Size ([ 100 , 100 ]) Hidden to Hidden Bias # Hidden --> Hidden BIAS(B3) list ( model . parameters ())[ 3 ] . size () torch . Size ([ 100 ]) Hidden to Output # Hidden --> Output (A2) list ( model . parameters ())[ 4 ] . size () torch . Size ([ 10 , 100 ]) Hidden to Output Bias # Hidden --> Output BIAS (B2) list ( model . parameters ())[ 5 ] . size () torch . Size ([ 10 ])","title":"Parameters In-Depth"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#step-7-train-model","text":"Process Convert inputs/labels to tensors with gradient accumulation abilities RNN Input: (1, 28) CNN Input: (1, 28, 28) FNN Input: (1, 28*28) Clear gradient buffets Get output given inputs Get loss Get gradients w.r.t. parameters Update parameters using gradients parameters = parameters - learning_rate * parameters_gradients REPEAT Same 7 step process for training models # Number of steps to unroll seq_dim = 28 iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): model . train () # Load images as tensors with gradient accumulation abilities images = images . view ( - 1 , seq_dim , input_dim ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits # outputs.size() --> 100, 10 outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : model . eval () # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Load images to a Torch tensors with gradient accumulation abilities images = images . view ( - 1 , seq_dim , input_dim ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 2.301494836807251 . Accuracy : 12 Iteration : 1000. Loss : 2.2986037731170654 . Accuracy : 14 Iteration : 1500. Loss : 2.278566598892212 . Accuracy : 18 Iteration : 2000. Loss : 2.169614315032959 . Accuracy : 21 Iteration : 2500. Loss : 1.1662731170654297 . Accuracy : 51 Iteration : 3000. Loss : 0.9290509223937988 . Accuracy : 71","title":"Step 7: Train Model"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#model-b-2-hidden-layer-relu","text":"Unroll 28 time steps Each step input size: 28 x 1 Total per unroll: 28 x 28 Feedforward Neural Network inpt size: 28 x 28 2 Hidden layer ReLU Activation Function","title":"Model B: 2 Hidden Layer (ReLU)"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#steps_1","text":"Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 2 Hidden Layer + ReLU import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class RNNModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , layer_dim , output_dim ): super ( RNNModel , self ) . __init__ () # Hidden dimensions self . hidden_dim = hidden_dim # Number of hidden layers self . layer_dim = layer_dim # Building your RNN # batch_first=True causes input/output tensors to be of shape # (batch_dim, seq_dim, feature_dim) self . rnn = nn . RNN ( input_dim , hidden_dim , layer_dim , batch_first = True , nonlinearity = 'relu' ) # Readout layer self . fc = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Initialize hidden state with zeros h0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () # We need to detach the hidden state to prevent exploding/vanishing gradients # This is part of truncated backpropagation through time (BPTT) out , hn = self . rnn ( x , h0 . detach ()) # Index hidden state of last time step # out.size() --> 100, 28, 100 # out[:, -1, :] --> 100, 100 --> just want last time step hidden states! out = self . fc ( out [:, - 1 , :]) # out.size() --> 100, 10 return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 hidden_dim = 100 layer_dim = 2 # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER output_dim = 10 model = RNNModel ( input_dim , hidden_dim , layer_dim , output_dim ) # JUST PRINTING MODEL & PARAMETERS print ( model ) print ( len ( list ( model . parameters ()))) for i in range ( len ( list ( model . parameters ()))): print ( list ( model . parameters ())[ i ] . size ()) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.01 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' # Number of steps to unroll seq_dim = 28 iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): model . train () # Load images as tensors with gradient accumulation abilities images = images . view ( - 1 , seq_dim , input_dim ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits # outputs.size() --> 100, 10 outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : model . eval () # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Resize images images = images . view ( - 1 , seq_dim , input_dim ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) RNNModel ( ( rnn ): RNN ( 28 , 100 , num_layers = 2 , batch_first = True ) ( fc ): Linear ( in_features = 100 , out_features = 10 , bias = True ) ) 10 torch . Size ([ 100 , 28 ]) torch . Size ([ 100 , 100 ]) torch . Size ([ 100 ]) torch . Size ([ 100 ]) torch . Size ([ 100 , 100 ]) torch . Size ([ 100 , 100 ]) torch . Size ([ 100 ]) torch . Size ([ 100 ]) torch . Size ([ 10 , 100 ]) torch . Size ([ 10 ]) Iteration : 500. Loss : 2.3019518852233887 . Accuracy : 11 Iteration : 1000. Loss : 2.299217700958252 . Accuracy : 11 Iteration : 1500. Loss : 2.279090166091919 . Accuracy : 14 Iteration : 2000. Loss : 2.126953125 . Accuracy : 25 Iteration : 2500. Loss : 1.356347680091858 . Accuracy : 57 Iteration : 3000. Loss : 0.7377720475196838 . Accuracy : 69 10 sets of parameters First hidden Layer \\(A_1 = [100, 28]\\) \\(A_3 = [100, 100]\\) \\(B_1 = [100]\\) \\(B_3 = [100]\\) Second hidden layer \\(A_2 = [100, 100]\\) \\(A_5 = [100, 100]\\) \\(B_2 = [100]\\) \\(B_5 = [100]\\) Readout layer \\(A_4 = [10, 100]\\) \\(B_4 = [10]\\)","title":"Steps"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#model-c-2-hidden-layer","text":"Unroll 28 time steps Each step input size: 28 x 1 Total per unroll: 28 x 28 Feedforward Neural Network inpt size: 28 x 28 2 Hidden layer Tanh Activation Function","title":"Model C: 2 Hidden Layer"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#steps_2","text":"Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model !!! \"2 Hidden + ReLU\" import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class RNNModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , layer_dim , output_dim ): super ( RNNModel , self ) . __init__ () # Hidden dimensions self . hidden_dim = hidden_dim # Number of hidden layers self . layer_dim = layer_dim # Building your RNN # batch_first=True causes input/output tensors to be of shape # (batch_dim, seq_dim, feature_dim) self . rnn = nn . RNN ( input_dim , hidden_dim , layer_dim , batch_first = True , nonlinearity = 'tanh' ) # Readout layer self . fc = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Initialize hidden state with zeros h0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . requires_grad_ () # One time step # We need to detach the hidden state to prevent exploding/vanishing gradients # This is part of truncated backpropagation through time (BPTT) out , hn = self . rnn ( x , h0 . detach ()) # Index hidden state of last time step # out.size() --> 100, 28, 100 # out[:, -1, :] --> 100, 100 --> just want last time step hidden states! out = self . fc ( out [:, - 1 , :]) # out.size() --> 100, 10 return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 hidden_dim = 100 layer_dim = 2 # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER output_dim = 10 model = RNNModel ( input_dim , hidden_dim , layer_dim , output_dim ) # JUST PRINTING MODEL & PARAMETERS print ( model ) print ( len ( list ( model . parameters ()))) for i in range ( len ( list ( model . parameters ()))): print ( list ( model . parameters ())[ i ] . size ()) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' # Number of steps to unroll seq_dim = 28 iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as tensors with gradient accumulation abilities images = images . view ( - 1 , seq_dim , input_dim ) . requires_grad_ () # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits # outputs.size() --> 100, 10 outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : # Resize images images = images . view ( - 1 , seq_dim , input_dim ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) RNNModel ( ( rnn ): RNN ( 28 , 100 , num_layers = 2 , batch_first = True ) ( fc ): Linear ( in_features = 100 , out_features = 10 , bias = True ) ) 10 torch . Size ([ 100 , 28 ]) torch . Size ([ 100 , 100 ]) torch . Size ([ 100 ]) torch . Size ([ 100 ]) torch . Size ([ 100 , 100 ]) torch . Size ([ 100 , 100 ]) torch . Size ([ 100 ]) torch . Size ([ 100 ]) torch . Size ([ 10 , 100 ]) torch . Size ([ 10 ]) Iteration : 500. Loss : 0.5943437218666077 . Accuracy : 77 Iteration : 1000. Loss : 0.22048641741275787 . Accuracy : 91 Iteration : 1500. Loss : 0.18479223549365997 . Accuracy : 94 Iteration : 2000. Loss : 0.2723771929740906 . Accuracy : 91 Iteration : 2500. Loss : 0.18817797303199768 . Accuracy : 92 Iteration : 3000. Loss : 0.1685929149389267 . Accuracy : 92","title":"Steps"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#summary-of-results","text":"Model A Model B Model C ReLU ReLU Tanh 1 Hidden Layer 2 Hidden Layers 2 Hidden Layers 100 Hidden Units 100 Hidden Units 100 Hidden Units 92.48% 95.09% 95.54%","title":"Summary of Results"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#general-deep-learning-notes","text":"2 ways to expand a recurrent neural network More non-linear activation units (neurons) More hidden layers Cons Need a larger dataset Curse of dimensionality Does not necessarily mean higher accuracy","title":"General Deep Learning Notes"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#3-building-a-recurrent-neural-network-with-pytorch-gpu","text":"","title":"3. Building a Recurrent Neural Network with PyTorch (GPU)"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#model-c-2-hidden-layer-tanh","text":"GPU: 2 things must be on GPU - model - tensors","title":"Model C: 2 Hidden Layer (Tanh)"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#steps_3","text":"Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model 2 Layer RNN + Tanh import torch import torch.nn as nn import torchvision.transforms as transforms import torchvision.datasets as dsets ''' STEP 1: LOADING DATASET ''' train_dataset = dsets . MNIST ( root = './data' , train = True , transform = transforms . ToTensor (), download = True ) test_dataset = dsets . MNIST ( root = './data' , train = False , transform = transforms . ToTensor ()) ''' STEP 2: MAKING DATASET ITERABLE ''' batch_size = 100 n_iters = 3000 num_epochs = n_iters / ( len ( train_dataset ) / batch_size ) num_epochs = int ( num_epochs ) train_loader = torch . utils . data . DataLoader ( dataset = train_dataset , batch_size = batch_size , shuffle = True ) test_loader = torch . utils . data . DataLoader ( dataset = test_dataset , batch_size = batch_size , shuffle = False ) ''' STEP 3: CREATE MODEL CLASS ''' class RNNModel ( nn . Module ): def __init__ ( self , input_dim , hidden_dim , layer_dim , output_dim ): super ( RNNModel , self ) . __init__ () # Hidden dimensions self . hidden_dim = hidden_dim # Number of hidden layers self . layer_dim = layer_dim # Building your RNN # batch_first=True causes input/output tensors to be of shape # (batch_dim, seq_dim, feature_dim) self . rnn = nn . RNN ( input_dim , hidden_dim , layer_dim , batch_first = True , nonlinearity = 'tanh' ) # Readout layer self . fc = nn . Linear ( hidden_dim , output_dim ) def forward ( self , x ): # Initialize hidden state with zeros ####################### # USE GPU FOR MODEL # ####################### h0 = torch . zeros ( self . layer_dim , x . size ( 0 ), self . hidden_dim ) . to ( device ) # One time step # We need to detach the hidden state to prevent exploding/vanishing gradients # This is part of truncated backpropagation through time (BPTT) out , hn = self . rnn ( x , h0 . detach ()) # Index hidden state of last time step # out.size() --> 100, 28, 100 # out[:, -1, :] --> 100, 100 --> just want last time step hidden states! out = self . fc ( out [:, - 1 , :]) # out.size() --> 100, 10 return out ''' STEP 4: INSTANTIATE MODEL CLASS ''' input_dim = 28 hidden_dim = 100 layer_dim = 2 # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER output_dim = 10 model = RNNModel ( input_dim , hidden_dim , layer_dim , output_dim ) ####################### # USE GPU FOR MODEL # ####################### device = torch . device ( \"cuda:0\" if torch . cuda . is_available () else \"cpu\" ) model . to ( device ) ''' STEP 5: INSTANTIATE LOSS CLASS ''' criterion = nn . CrossEntropyLoss () ''' STEP 6: INSTANTIATE OPTIMIZER CLASS ''' learning_rate = 0.1 optimizer = torch . optim . SGD ( model . parameters (), lr = learning_rate ) ''' STEP 7: TRAIN THE MODEL ''' # Number of steps to unroll seq_dim = 28 iter = 0 for epoch in range ( num_epochs ): for i , ( images , labels ) in enumerate ( train_loader ): # Load images as tensors with gradient accumulation abilities ####################### # USE GPU FOR MODEL # ####################### images = images . view ( - 1 , seq_dim , input_dim ) . requires_grad_ () . to ( device ) labels = labels . to ( device ) # Clear gradients w.r.t. parameters optimizer . zero_grad () # Forward pass to get output/logits # outputs.size() --> 100, 10 outputs = model ( images ) # Calculate Loss: softmax --> cross entropy loss loss = criterion ( outputs , labels ) # Getting gradients w.r.t. parameters loss . backward () # Updating parameters optimizer . step () iter += 1 if iter % 500 == 0 : # Calculate Accuracy correct = 0 total = 0 # Iterate through test dataset for images , labels in test_loader : ####################### # USE GPU FOR MODEL # ####################### images = images . view ( - 1 , seq_dim , input_dim ) . to ( device ) # Forward pass only to get logits/output outputs = model ( images ) # Get predictions from the maximum value _ , predicted = torch . max ( outputs . data , 1 ) # Total number of labels total += labels . size ( 0 ) # Total correct predictions ####################### # USE GPU FOR MODEL # ####################### if torch . cuda . is_available (): correct += ( predicted . cpu () == labels . cpu ()) . sum () else : correct += ( predicted == labels ) . sum () accuracy = 100 * correct / total # Print Loss print ( 'Iteration: {} . Loss: {} . Accuracy: {} ' . format ( iter , loss . item (), accuracy )) Iteration : 500. Loss : 0.5983774662017822 . Accuracy : 81 Iteration : 1000. Loss : 0.2960105836391449 . Accuracy : 86 Iteration : 1500. Loss : 0.19428101181983948 . Accuracy : 93 Iteration : 2000. Loss : 0.11918395012617111 . Accuracy : 95 Iteration : 2500. Loss : 0.11246936023235321 . Accuracy : 95 Iteration : 3000. Loss : 0.15849310159683228 . Accuracy : 95","title":"Steps"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#summary","text":"We've learnt to... Success Feedforward Neural Networks Transition to Recurrent Neural Networks RNN Models in PyTorch Model A: 1 Hidden Layer RNN (ReLU) Model B: 2 Hidden Layer RNN (ReLU) Model C: 2 Hidden Layer RNN (Tanh) Models Variation in Code Modifying only step 4 Ways to Expand Model\u2019s Capacity More non-linear activation units ( neurons ) More hidden layers Cons of Expanding Capacity Need more data Does not necessarily mean higher accuracy GPU Code 2 things on GPU model tensors with gradient accumulation abilities Modifying only Step 3, 4 and 7 7 Step Model Building Recap Step 1: Load Dataset Step 2: Make Dataset Iterable Step 3: Create Model Class Step 4: Instantiate Model Class Step 5: Instantiate Loss Class Step 6: Instantiate Optimizer Class Step 7: Train Model Step 7: Train Model","title":"Summary"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#citation","text":"If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.","title":"Citation"},{"location":"deep_learning/production_pytorch/speed_optimization_basics_numba/","text":"Speed Optimization Basics: Numba \u00b6 When to use Numba \u00b6 Numba works well when the code relies a lot on (1) numpy, (2) loops, and/or (2) cuda. Hence, we would like to maximize the use of numba in our code where possible where there are loops/numpy Numba CPU: nopython \u00b6 For a basic numba application, we can cecorate python function thus allowing it to run without python interpreter Essentially, it will compile the function with specific arguments once into machine code, then uses the cache subsequently With Numba: no python \u00b6 from numba import jit , prange import numpy as np # Numpy array of 10k elements input_ndarray = np . random . rand ( 10000 ) . reshape ( 10000 ) # This is the only extra line of code you need to add # which is a decorator @jit ( nopython = True ) def go_fast ( a ): trace = 0 for i in range ( a . shape [ 0 ]): trace += np . tanh ( a [ i ]) return a + trace % timeit go_fast ( input_ndarray ) 161 \u00b5s \u00b1 2.62 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 10000 loops each) Without numba \u00b6 This is much slower, time measured in the millisecond space rather than microsecond with @jit(nopython=True) or @njit # Without numba: notice how this is really slow def go_normal ( a ): trace = 0 for i in range ( a . shape [ 0 ]): trace += np . tanh ( a [ i ]) return a + trace % timeit go_normal ( input_ndarray ) 10.5 ms \u00b1 163 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 100 loops each) Numba CPU: parallel \u00b6 Here, instead of the normal range() function we would use for loops, we would need to use prange() which allows us to execute the loops in parallel on separate threads As you can see, it's slightly faster than @jit(nopython=True) @jit ( nopython = True , parallel = True ) def go_even_faster ( a ): trace = 0 for i in prange ( a . shape [ 0 ]): trace += np . tanh ( a [ i ]) return a + trace % timeit go_even_faster ( input_ndarray ) 148 \u00b5s \u00b1 71.3 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1 loop each) Numba CPU: fastmath \u00b6 What if we relax our condition of strictly adhering to IEEE 754 . We can have faster performance (depends) I would say this is the least additional speed-up unless you really dig into areas where fastmath=True thrives @jit ( nopython = True , parallel = True , fastmath = True ) def go_super_fast ( a ): trace = 0 for i in prange ( a . shape [ 0 ]): trace += np . tanh ( a [ i ]) return a + trace % timeit go_super_fast ( input_ndarray ) 113 \u00b5s \u00b1 39.7 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1 loop each) Summary \u00b6 When to use Numba (1) numpy array or torch tensors, (2) loops, and/or (3) cuda Numba CPU: nopython\u00b6 Numba CPU: parallel Numba CPU: fastmath","title":"Speed Optimization Basics Numba"},{"location":"deep_learning/production_pytorch/speed_optimization_basics_numba/#speed-optimization-basics-numba","text":"","title":"Speed Optimization Basics: Numba"},{"location":"deep_learning/production_pytorch/speed_optimization_basics_numba/#when-to-use-numba","text":"Numba works well when the code relies a lot on (1) numpy, (2) loops, and/or (2) cuda. Hence, we would like to maximize the use of numba in our code where possible where there are loops/numpy","title":"When to use Numba"},{"location":"deep_learning/production_pytorch/speed_optimization_basics_numba/#numba-cpu-nopython","text":"For a basic numba application, we can cecorate python function thus allowing it to run without python interpreter Essentially, it will compile the function with specific arguments once into machine code, then uses the cache subsequently","title":"Numba CPU: nopython"},{"location":"deep_learning/production_pytorch/speed_optimization_basics_numba/#with-numba-no-python","text":"from numba import jit , prange import numpy as np # Numpy array of 10k elements input_ndarray = np . random . rand ( 10000 ) . reshape ( 10000 ) # This is the only extra line of code you need to add # which is a decorator @jit ( nopython = True ) def go_fast ( a ): trace = 0 for i in range ( a . shape [ 0 ]): trace += np . tanh ( a [ i ]) return a + trace % timeit go_fast ( input_ndarray ) 161 \u00b5s \u00b1 2.62 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 10000 loops each)","title":"With Numba: no python"},{"location":"deep_learning/production_pytorch/speed_optimization_basics_numba/#without-numba","text":"This is much slower, time measured in the millisecond space rather than microsecond with @jit(nopython=True) or @njit # Without numba: notice how this is really slow def go_normal ( a ): trace = 0 for i in range ( a . shape [ 0 ]): trace += np . tanh ( a [ i ]) return a + trace % timeit go_normal ( input_ndarray ) 10.5 ms \u00b1 163 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 100 loops each)","title":"Without numba"},{"location":"deep_learning/production_pytorch/speed_optimization_basics_numba/#numba-cpu-parallel","text":"Here, instead of the normal range() function we would use for loops, we would need to use prange() which allows us to execute the loops in parallel on separate threads As you can see, it's slightly faster than @jit(nopython=True) @jit ( nopython = True , parallel = True ) def go_even_faster ( a ): trace = 0 for i in prange ( a . shape [ 0 ]): trace += np . tanh ( a [ i ]) return a + trace % timeit go_even_faster ( input_ndarray ) 148 \u00b5s \u00b1 71.3 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1 loop each)","title":"Numba CPU: parallel"},{"location":"deep_learning/production_pytorch/speed_optimization_basics_numba/#numba-cpu-fastmath","text":"What if we relax our condition of strictly adhering to IEEE 754 . We can have faster performance (depends) I would say this is the least additional speed-up unless you really dig into areas where fastmath=True thrives @jit ( nopython = True , parallel = True , fastmath = True ) def go_super_fast ( a ): trace = 0 for i in prange ( a . shape [ 0 ]): trace += np . tanh ( a [ i ]) return a + trace % timeit go_super_fast ( input_ndarray ) 113 \u00b5s \u00b1 39.7 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1 loop each)","title":"Numba CPU: fastmath"},{"location":"deep_learning/production_pytorch/speed_optimization_basics_numba/#summary","text":"When to use Numba (1) numpy array or torch tensors, (2) loops, and/or (3) cuda Numba CPU: nopython\u00b6 Numba CPU: parallel Numba CPU: fastmath","title":"Summary"},{"location":"machine_learning/intro/","text":"Machine Learning \u00b6 We'll be covering both CPU and GPU implementations of machine learning algorithms and data wrangling libraries. Packages and Languages you will Learn to Use Python RAPIDS cuDF RAPIDS cuML Scikit-learn Pandas NumPy CuPy Work in progress This open-source portion is still a work in progress. Stay tuned while we gradually upload our tutorials and notes. Feel free to contact Ritchie Ng if you would like to contribute via our Facebook page.","title":"Introduction"},{"location":"machine_learning/intro/#machine-learning","text":"We'll be covering both CPU and GPU implementations of machine learning algorithms and data wrangling libraries. Packages and Languages you will Learn to Use Python RAPIDS cuDF RAPIDS cuML Scikit-learn Pandas NumPy CuPy Work in progress This open-source portion is still a work in progress. Stay tuned while we gradually upload our tutorials and notes. Feel free to contact Ritchie Ng if you would like to contribute via our Facebook page.","title":"Machine Learning"},{"location":"machine_learning/gpu/gpu_fractional_differencing/","text":"GFD: GPU Fractional Differencing \u00b6 Summary \u00b6 Typically we attempt to achieve some form of stationarity via a transformation on our time series through common methods including integer differencing. However, integer differencing unnecessarily removes too much memory to achieve stationarity. An alternative, fractional differencing, allows us to achieve stationarity while maintaining the maximum amount of memory compared to integer differencing. While existing CPU-based implementations are inefficient for running fractional differencing on many large-scale time series, our GPU-based implementation enables rapid fractional differencing. This tutorial walks you through our GPU implementation of fractional differencing (GFD). Also, this tutorial is a special collaboration between industry and academia led by me. Credits \u00b6 Before we start, I would like to thank and give credits for this particular post to: 1. NVIDIA (Ettikan, Chris and Nick), Amazon AWS, ensemblecap.ai, and NExT++ (NUS School of Computing) 2. Marcos Lopez de Prado for his recent push on the use of fractional differencing of which this guide is based on. 3. Hosking for his paper in 1981 on fractional differencing. Links \u00b6 Code Repository: https://github.com/ritchieng/fractional_differencing_gpu Presentation: https://www.researchgate.net/publication/335159299_GFD_GPU_Fractional_Differencing_for_Rapid_Large-scale_Stationarizing_of_Time_Series_Data_while_Minimizing_Memory_Loss Installation of Libraries (Optional) \u00b6 These steps are for easily running RAPIDS in Google Colab. You can skip this section if you've RAPIDS installed. import pynvml pynvml . nvmlInit () handle = pynvml . nvmlDeviceGetHandleByIndex ( 0 ) device_name = pynvml . nvmlDeviceGetName ( handle ) print ( f 'Currently running on { device_name } ' ) if device_name != b 'Tesla T4' : raise Exception ( \"\"\" Unfortunately this instance does not have a T4 GPU. Please make sure you've configured Colab to request a GPU instance type. Sometimes Colab allocates a Tesla K80 instead of a T4. Resetting the instance. If you get a K80 GPU, try Runtime -> Reset all runtimes -> Keep trying! \"\"\" ) else : print ( 'Woo! You got the right kind of GPU, a Tesla T4!' ) Currently running on b'Tesla T4' Woo! You got the right kind of GPU, a Tesla T4! # intall miniconda ! wget - c https : // repo . continuum . io / miniconda / Miniconda3 - 4.5.4 - Linux - x86_64 . sh ! chmod + x Miniconda3 - 4.5.4 - Linux - x86_64 . sh ! bash ./ Miniconda3 - 4.5.4 - Linux - x86_64 . sh - b - f - p / usr / local # install RAPIDS packages ! conda install - q - y -- prefix / usr / local - c conda - forge \\ - c rapidsai - nightly / label / cuda10 .0 - c nvidia / label / cuda10 .0 \\ cudf cuml # set environment vars import sys , os , shutil sys . path . append ( '/usr/local/lib/python3.6/site-packages/' ) os . environ [ 'NUMBAPRO_NVVM' ] = '/usr/local/cuda/nvvm/lib64/libnvvm.so' os . environ [ 'NUMBAPRO_LIBDEVICE' ] = '/usr/local/cuda/nvvm/libdevice/' # copy .so files to current working dir for fn in [ 'libcudf.so' , 'librmm.so' ]: shutil . copy ( '/usr/local/lib/' + fn , os . getcwd ()) --2019-08-19 05:36:25-- https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh Resolving repo.continuum.io (repo.continuum.io)... 104.18.200.79, 104.18.201.79, 2606:4700::6812:c84f, ... Connecting to repo.continuum.io (repo.continuum.io)|104.18.200.79|:443... connected. HTTP request sent, awaiting response... 416 Requested Range Not Satisfiable The file is already fully retrieved; nothing to do. PREFIX=/usr/local installing: python-3.6.5-hc3d631a_2 ... Python 3.6.5 :: Anaconda, Inc. installing: ca-certificates-2018.03.07-0 ... installing: conda-env-2.6.0-h36134e3_1 ... installing: libgcc-ng-7.2.0-hdf63c60_3 ... installing: libstdcxx-ng-7.2.0-hdf63c60_3 ... installing: libffi-3.2.1-hd88cf55_4 ... installing: ncurses-6.1-hf484d3e_0 ... installing: openssl-1.0.2o-h20670df_0 ... installing: tk-8.6.7-hc745277_3 ... installing: xz-5.2.4-h14c3975_4 ... installing: yaml-0.1.7-had09818_2 ... installing: zlib-1.2.11-ha838bed_2 ... installing: libedit-3.1.20170329-h6b74fdf_2 ... installing: readline-7.0-ha6073c6_4 ... installing: sqlite-3.23.1-he433501_0 ... installing: asn1crypto-0.24.0-py36_0 ... installing: certifi-2018.4.16-py36_0 ... installing: chardet-3.0.4-py36h0f667ec_1 ... installing: idna-2.6-py36h82fb2a8_1 ... installing: pycosat-0.6.3-py36h0a5515d_0 ... installing: pycparser-2.18-py36hf9f622e_1 ... installing: pysocks-1.6.8-py36_0 ... installing: ruamel_yaml-0.15.37-py36h14c3975_2 ... installing: six-1.11.0-py36h372c433_1 ... installing: cffi-1.11.5-py36h9745a5d_0 ... installing: setuptools-39.2.0-py36_0 ... installing: cryptography-2.2.2-py36h14c3975_0 ... installing: wheel-0.31.1-py36_0 ... installing: pip-10.0.1-py36_0 ... installing: pyopenssl-18.0.0-py36_0 ... installing: urllib3-1.22-py36hbe7ace6_0 ... installing: requests-2.18.4-py36he2e5f8d_1 ... installing: conda-4.5.4-py36_0 ... unlinking: ca-certificates-2019.6.16-hecc5488_0 unlinking: certifi-2019.6.16-py36_1 unlinking: conda-4.7.11-py36_0 unlinking: cryptography-2.7-py36h72c5cf5_0 unlinking: libgcc-ng-9.1.0-hdf63c60_0 unlinking: libstdcxx-ng-9.1.0-hdf63c60_0 unlinking: openssl-1.1.1c-h516909a_0 unlinking: python-3.6.7-h381d211_1004 unlinking: sqlite-3.28.0-h8b20d00_0 unlinking: tk-8.6.9-hed695b0_1002 installation finished. WARNING: You currently have a PYTHONPATH environment variable set. This may cause unexpected behavior when running the Python interpreter in Miniconda3. For best results, please verify that your PYTHONPATH only points to directories of packages that are compatible with the Python interpreter in Miniconda3: /usr/local Solving environment: ...working... done ## Package Plan ## environment location: /usr/local added / updated specs: - cudf - cuml The following packages will be UPDATED: ca-certificates: 2018.03.07-0 --> 2019.6.16-hecc5488_0 conda-forge certifi: 2018.4.16-py36_0 --> 2019.6.16-py36_1 conda-forge conda: 4.5.4-py36_0 --> 4.7.11-py36_0 conda-forge cryptography: 2.2.2-py36h14c3975_0 --> 2.7-py36h72c5cf5_0 conda-forge libgcc-ng: 7.2.0-hdf63c60_3 --> 9.1.0-hdf63c60_0 libstdcxx-ng: 7.2.0-hdf63c60_3 --> 9.1.0-hdf63c60_0 openssl: 1.0.2o-h20670df_0 --> 1.1.1c-h516909a_0 conda-forge python: 3.6.5-hc3d631a_2 --> 3.6.7-h381d211_1004 conda-forge sqlite: 3.23.1-he433501_0 --> 3.28.0-h8b20d00_0 conda-forge tk: 8.6.7-hc745277_3 --> 8.6.9-hed695b0_1002 conda-forge Preparing transaction: ...working... done Verifying transaction: ...working... done Executing transaction: ...working... done You would need to do a Runtime > Restart and Run All on your Google Colab notebook at this step. Imports \u00b6 # Critical imports for GPU cuDF import nvstrings , nvcategory , cudf # Other imports import numpy as np import pandas as pd import time import pandas_datareader.data as web from datetime import datetime from matplotlib import pyplot as plt from numba import cuda Plotting Style \u00b6 # Display settings: just to make it slightly prettier # There'll be a separate post on beautiful plots % matplotlib inline figsize = ( 25 , 6 ) plt . style . use ( 'fivethirtyeight' ) Necessity of Fractional Differencing \u00b6 Before we get into why fractional differencing is critical, let us inspect a simple time series pulled from the Fed's database: S&P 500. Pulling, Processing & Plotting SPX Time Series \u00b6 # Read SPX data 2010-2019 from FED database: https://fred.stlouisfed.org/categories/94 asset_name = 'SPX' start = datetime ( 2010 , 1 , 1 ) end = datetime ( 2019 , 8 , 1 ) df_raw = web . DataReader ( 'SP500' , 'fred' , start , end ) # Basic clean up data: dropna (there are other more robust treatments) df_raw = df_raw . dropna () df_raw . columns = [ asset_name ] df_raw . plot ( figsize = figsize ) plt . title ( 'S&P 500 Absolute Levels' ); Integer Differencing (1 Day Returns) SPX Time Series \u00b6 Traditionally we typically might difference our time series by one day (daily returns) or more to reach some form of stationarity via tests like ADF, KPSS, PP and more. However, these forms of integer differencing causes more information than needed to be lost. # One day returns calculation through differencing by 1 differencing_factor = 1 df_daily_returns = ( df_raw - df_raw . shift ( differencing_factor )) / df_raw . shift ( differencing_factor ) * 100 df_daily_returns . plot ( figsize = figsize ) plt . title ( f ' { asset_name } Daily Returns via Differencing 1 Day' ); Integer Differencing Causes Unnecessary Memory Loss \u00b6 Because of this reason, we propose fractional differencing to achieve stationarity while minimizing memory loss. Basic Fractional Differencing \u00b6 Fractional differencing allows us to achieve stationarity while maintaining the maximum amount of memory compared to integer differencing. This was originally introduced in 1981 in his paper \u201cFractional Differencing\u201d by J. R. M. Hosking1 and subsequent work by others concentrated on fast and efficient implementations for fractional differentiation for continuous stochastic processes. Recently, fractional differencing was introduced for financial time series through the fixed window fractional differencing instead of the expanding window method by Marcos Lopez de Prado2. Fractional Differencing Weight Function and Plot \u00b6 Weights formula: \\(w_k = -w_{k-1} \\frac{d - k + 1}{k}\\) Weight converges to zero: \\(w_k \\rightarrow 0\\) def get_weights ( d , num_k ): r \"\"\"Calculate weights ($w$) for each lag ($k$) through $w_k = -w_{k-1} \\frac{d - k + 1}{k}$. Args: d (int): differencing value. num_k (int): number of lags (typically length of timeseries) to calculate w. \"\"\" w_k = np . array ([ 1 ]) for k in range ( 1 , num_k ): w_k = np . append ( w_k , - w_k [ - 1 ] * (( d - k + 1 )) / k ) w_k = w_k . reshape ( - 1 , 1 ) return w_k def plot_weights ( range_d , num_k , num_d_interval ): r \"\"\"Plot weights ($w$) for each lag ($k$) for varying differencing value ($d$). Args: range_d (list): range of differencing values to plot. num_k (int): number of lags (typically length of timeseries) to plot. num_d_interval (int): number of d interval. \"\"\" # Get differencing values interval = np . linspace ( range_d [ 0 ], range_d [ 1 ], num_d_interval ) # Dataframe of lags (rows) x number of differencing intervals (columns) df_wk = pd . DataFrame ( np . zeros (( num_k , num_d_interval ))) # Get weights array per differencing value for i , d in enumerate ( interval ): df_wk [ i ] = get_weights ( d , num_k ) # Rename columns for legend df_wk . columns = [ round ( x , 1 ) for x in interval ] # Plot df_wk . plot ( figsize = figsize ) plt . title ( 'Lag weights ($w_k$) for various differencing values (d)' ) plt . legend ( title = 'Differencing value (d)' ) plt . ylabel ( 'Weights (w)' ) plt . xlabel ( 'Lag (k)' ) plt . show () # Return weights return df_wk df_wk = plot_weights ( range_d = [ 0 , 1 ], num_k = 7 , num_d_interval = 6 ) Fractional Differencing Weight Function Table \u00b6 # Dataframe for lag weights (w) for various differencing values (d) df_wk .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } 0.0 0.2 0.4 0.6 0.8 1.0 0 1.0 1.000000 1.000000 1.000000 1.000000 1.0 1 -0.0 -0.200000 -0.400000 -0.600000 -0.800000 -1.0 2 -0.0 -0.080000 -0.120000 -0.120000 -0.080000 0.0 3 -0.0 -0.048000 -0.064000 -0.056000 -0.032000 0.0 4 -0.0 -0.033600 -0.041600 -0.033600 -0.017600 0.0 5 -0.0 -0.025536 -0.029952 -0.022848 -0.011264 0.0 6 -0.0 -0.020429 -0.022963 -0.016755 -0.007885 0.0 Fractional Differencing Value Calculation Toy Example \u00b6 Assume spot have values 100, 99, 98, 97, 96, 95, 94 from k=0 to k=6 For \\(d = 0\\) (no differencing) \u00b6 differenced_value = 100 * 1 + 99 * 0 + 98 * 0 + 97 * 0 + 96 * 0 + 95 * 0 + 94 * 0 print ( f 'Differenced value: { differenced_value } ' ) df_wk . iloc [:, 0 ] Differenced value: 100 0 1.0 1 -0.0 2 -0.0 3 -0.0 4 -0.0 5 -0.0 6 -0.0 Name: 0.0, dtype: float64 For \\(d = 0.4\\) (fractional differencing) \u00b6 differenced_value = 100 * 1 + 99 * - 0.400000 + 98 * - 0.120000 + 97 * - 0.064000 + 96 * - 0.041600 + 95 * - 0.041600 + 94 * - 0.041600 print ( f 'Differenced value: { differenced_value } ' ) df_wk . iloc [:, 1 ] Differenced value: 30.576000000000004 0 1.000000 1 -0.200000 2 -0.080000 3 -0.048000 4 -0.033600 5 -0.025536 6 -0.020429 Name: 0.2, dtype: float64 For \\(d = 1\\) (integer differencing) \u00b6 differenced_value = 100 * 1 + 99 * - 1 + 98 * 0 + 97 * 0 + 96 * 0 + 95 * 0 + 94 * 0 print ( f 'Differenced value: { differenced_value } ' ) df_wk . iloc [:, - 1 ] Differenced value: 1 0 1.0 1 -1.0 2 0.0 3 0.0 4 0.0 5 0.0 6 0.0 Name: 1.0, dtype: float64 Cumulative Sum of Weights Behavior \u00b6 Higher differencing value would accelerate cumulative sum weight decay This causes: Less information taken from further lags More information loss from further lags df_wk . cumsum () . plot ( figsize = figsize ) plt . title ( 'Cumulative Sum of Lag weights ($\\sum_0^k w_k$) for various differencing values (d)' ) plt . legend ( title = 'Differencing value ($d$)' ) plt . ylabel ( 'Cumulative Sum of Weights ($\\sum_0^k w_k$)' ) plt . xlabel ( 'Lag (k)' ); Fixed Window Fractional Differencing (CPU) \u00b6 Floored Weights Function \u00b6 For computational efficiency, we want to stop the calculation of weights when the weights are too small (below a certain small predetermined small value). Image we had a time series with 1,000,000 data points, we do not want to compute the weights up till 1m lag for our latest data point! We can simply calculate the weights till they're too small which is what this function does. def get_weights_floored ( d , num_k , floor = 1e-3 ): r \"\"\"Calculate weights ($w$) for each lag ($k$) through $w_k = -w_{k-1} \\frac{d - k + 1}{k}$ provided weight above a minimum value (floor) for the weights to prevent computation of weights for the entire time series. Args: d (int): differencing value. num_k (int): number of lags (typically length of timeseries) to calculate w. floor (float): minimum value for the weights for computational efficiency. \"\"\" w_k = np . array ([ 1 ]) k = 1 while k < num_k : w_k_latest = - w_k [ - 1 ] * (( d - k + 1 )) / k if abs ( w_k_latest ) <= floor : break w_k = np . append ( w_k , w_k_latest ) k += 1 w_k = w_k . reshape ( - 1 , 1 ) return w_k # Show large time series being stopped # Notice how max lag is at 9? d = 0.5 num_k = 1000 weights = get_weights_floored ( d = d , num_k = num_k ) pd . DataFrame ( weights ) . plot ( legend = False , figsize = figsize ) # Plot plt . title ( f 'Lag weights ($w_k$) for differencing value $d= { d } $' ) plt . ylabel ( 'Weights (w)' ) plt . xlabel ( 'Lag (k)' ); Fixed Window Fractional Differencing Function (CPU) \u00b6 Here, we use the new floored weights function compared to our normal weights function Calculate our differenced values through the dot product of our transposed weights matrix and our original time series within that window Keep repeating by shifting the window by one time step until the end of the time series def frac_diff ( df , d , floor = 1e-3 ): r \"\"\"Fractionally difference time series via CPU. Args: df (pd.DataFrame): dataframe of raw time series values. d (float): differencing value from 0 to 1 where > 1 has no FD. floor (float): minimum value of weights, ignoring anything smaller. \"\"\" # Get weights window weights = get_weights_floored ( d = d , num_k = len ( df ), floor = floor ) weights_window_size = len ( weights ) # Reverse weights weights = weights [:: - 1 ] # Blank fractionally differenced series to be filled df_fd = [] # Slide window of time series, to calculated fractionally differenced values # per window for idx in range ( weights_window_size , df . shape [ 0 ]): # Dot product of weights and original values # to get fractionally differenced values date_idx = df . index [ idx ] df_fd . append ( np . dot ( weights . T , df . iloc [ idx - weights_window_size : idx ]) . item ()) # Return FD values and weights df_fd = pd . DataFrame ( df_fd ) return df_fd , weights # Start timer start = time . time () df_raw_fd , weights = frac_diff ( df_raw , d = 0.5 , floor = 5e-5 ) # End timer end = time . time () print ( f 'Time { end - start } s' ) # Plot df_raw_fd . plot ( figsize = figsize ); Time 0.5417799949645996 s import multiprocessing multiprocessing . cpu_count () 4 Existing CPU-based implementations are inefficient for running fractional differencing on many large-scale time-series. GPU-based implementations provide an avenue to adapt to this century\u2019s big data requirements. Fixed Window Fractional Differencing Function (GPU) \u00b6 This gives a slight 1.5x to 2x speed-up on an NVIDIA T4 on Google Colab for this tiny dataset For larger time series in line with real-world situations, it can easily be up to 100x to 1000x dataset (shown below). def moving_dot_product_kernel ( in_data , out , window_size , weights ): # Set the first window_size-1 rows in each chunk to np.nan due # insufficient history for i in range ( cuda . threadIdx . x , window_size - 1 , cuda . blockDim . x ): out [ i ] = np . nan # Compute dot product of preceding window_size rows for i in range ( cuda . threadIdx . x + window_size - 1 , in_data . size , cuda . blockDim . x ): rolling_dot_product = 0.0 k = 0 for j in range ( i - window_size + 1 , i + 1 ): rolling_dot_product += in_data [ j ] * weights [ k ][ 0 ] k += 1 out [ i ] = rolling_dot_product def frac_diff_gpu ( df , d , floor = 1e-3 ): r \"\"\"Fractionally difference time series via GPU. Args: df (pd.DataFrame): dataframe of raw time series values. d (float): differencing value from 0 to 1 where > 1 has no FD. floor (float): minimum value of weights, ignoring anything smaller. \"\"\" # Bring dataframe to GPU, reset index for GPU dot product kernel gdf_raw = cudf . from_pandas ( df ) . reset_index ( drop = True ) gdf_raw . columns = [ 'in_data' ] # Get weights window weights = get_weights_floored ( d = d , num_k = len ( gdf_raw ), floor = floor ) weights_window_size = len ( weights ) # Reverse weights and as contiguous weights = np . ascontiguousarray ( weights [:: - 1 ]) # Bring weights to GPU gdf_weights = cudf . DataFrame () gdf_weights [ gdf_raw . columns [ 0 ]] = weights . reshape ( - 1 ) # Length of data data_length = len ( gdf_raw ) # T4: max of 518 threads per block. # V100: max 1024 threads per block threads_per_block = 518 # Chunk size split # This has to be improved, but as a v0.1, it's sufficient to show speed-up # Up to easily 100 million data points trunk_size = data_length # Get fractionally differenced time series through GPU function gdf_raw_fd = gdf_raw . apply_chunks ( moving_dot_product_kernel , incols = [ 'in_data' ], outcols = dict ( out = np . float64 ), kwargs = dict ( window_size = weights_window_size , weights = weights ), chunks = list ( range ( 0 , data_length , trunk_size )) + [ data_length ], tpb = threads_per_block ) # Bring to CPU for normal manipulation df_raw_fd = gdf_raw_fd . to_pandas () . dropna () . iloc [: - 1 , 1 ] return df_raw_fd , weights # Start timer start = time . time () df_raw_fd_from_gpu , weights = frac_diff_gpu ( df_raw , d = 0.5 , floor = 5e-5 ) # End timer end = time . time () print ( f 'Time { end - start } s' ) # Plot df_raw_fd_from_gpu . plot ( figsize = figsize ); Time 0.27262187004089355 s Check Values \u00b6 # Compare tail values print ( 'Tail values check' ) print ( df_raw_fd_from_gpu . tail () . values ) print ( df_raw_fd . tail () . values . reshape ( - 1 ,)) Tail values check [107.00340988 133.85520208 117.92878691 109.44132697 79.32562638] [107.00340988 133.85520208 117.92878691 109.44132697 79.32562638] # Compare tail values print ( 'Head values check' ) print ( df_raw_fd_from_gpu . head () . values ) print ( df_raw_fd . head () . values . reshape ( - 1 ,)) Head values check [56.74989213 52.56766288 47.32421832 45.89772154 37.74401501] [56.74989213 52.56766288 47.32421832 45.89772154 37.74401501] num_rows_true = ( df_raw_fd . values . astype ( np . float32 ) == df_raw_fd_from_gpu . values . astype ( np . float32 )) . astype ( int ) . sum () total_rows = df_raw_fd . shape [ 0 ] print ( f 'Number of rows equal: { num_rows_true } ' ) print ( f 'Total number of rows: { total_rows } ' ) Number of rows equal: 2093 Total number of rows: 2093 Stationarity \u00b6 \"A stationary time series is one whose statistical properties such as mean, variance, autocorrelation, etc. are all constant over time\", Robert Nau, Duke University Essentially what we are trying to do here with fractional differencing is to attempt to have a stationary time series without losing too much memory. There are many ways to check if a time series is stationary, but we will be using 1 test here (they're more, but it's not the purpose of this tutorial) Augmented Dickey\u2013Fuller (ADF) test: check for unit root ADF Test \u00b6 # Import adf/kpss from statsmodels.tsa.stattools import adfuller import warnings warnings . filterwarnings ( \"ignore\" ) # Ignore deprecations for cleaner output ADF: raw data (d=0) \u00b6 # ADF on raw data print ( 'Stationarity Test for SPX Absolute Levels (d=0)' ) print ( '-' * 50 ) result = adfuller ( df_raw [ 'SPX' ], regression = 'c' ) print ( f 't-stat \\n\\t { result [ 0 ] : .2f } ' ) print ( f 'p-value \\n\\t { result [ 1 ] : .2f } ' ) print ( f 'Critical Values' ) for key , value in result [ 4 ] . items (): print ( f ' \\t { key } : { value : .2f } ' ) # test statistic > test statistic (1%) AND p-value > 0.01 # Fail to reject the null hypothesis that there's unit root at the 1% significance level # ==> Fail to reject H0 ==> Unit root ==> Non-stationary Stationarity Test for SPX Absolute Levels (d=0) -------------------------------------------------- t-stat -0.11 p-value 0.95 Critical Values 1%: -3.43 5%: -2.86 10%: -2.57 ADF: daily return data (d=1) \u00b6 # ADF on 1 day returns (d=1) print ( 'Stationarity Test for SPX Daily Returns (d=1)' ) print ( '-' * 50 ) result = adfuller ( df_raw [ 'SPX' ] . pct_change ( 1 ) . dropna (), regression = 'c' ) print ( f 't-stat \\n\\t { result [ 0 ] : .2f } ' ) print ( f 'p-value \\n\\t { result [ 1 ] : .2f } ' ) print ( f 'Critical Values' ) for key , value in result [ 4 ] . items (): print ( f ' \\t { key } : { value : .2f } ' ) # test statistic < test statistic (1%) AND p-value < 0.01 # Reject the null hypothesis that there's unit root at the 1% significance level # ==> Reject H0 ==> No unit root ==> Stationary Stationarity Test for SPX Daily Returns (d=1) -------------------------------------------------- t-stat -11.12 p-value 0.00 Critical Values 1%: -3.43 5%: -2.86 10%: -2.57 ADF: fractionally differenced data (d=0.5) \u00b6 # ADF on fractionally differenced values print ( 'Stationarity Test (ADF) for SPX Fractionally Differenced (d=0.5)' ) print ( '-' * 50 ) df_raw_fd_from_gpu , weights = frac_diff_gpu ( df_raw , d = 0.5 , floor = 5e-5 ) result = adfuller ( df_raw_fd_from_gpu , regression = 'c' ) print ( f 't-stat \\n\\t { result [ 0 ] : .2f } ' ) print ( f 'p-value \\n\\t { result [ 1 ] : .2f } ' ) print ( f 'Critical Values' ) for key , value in result [ 4 ] . items (): print ( f ' \\t { key } : { value : .2f } ' ) # test statistic < test statistic (1%) AND p-value < 0.01 # Reject the null hypothesis that there's unit root at the 1% significance level # ==> Reject H0 ==> No unit root ==> Stationary Stationarity Test (ADF) for SPX Fractionally Differenced (d=0.5) -------------------------------------------------- t-stat -3.86 p-value 0.00 Critical Values 1%: -3.43 5%: -2.86 10%: -2.57 Large-scale Rapid Fractional Differencing \u00b6 Here we test on a 100k, 1m, 10m and 100m datapoint dataframe. # Create 100m data points large_time_series_length = int ( 1e6 ) print ( large_time_series_length ) 1000000 # Start timer start = time . time () df_raw = pd . DataFrame ( np . arange ( large_time_series_length ) * np . random . rand ( large_time_series_length )) df_raw_fd_from_gpu , weights = frac_diff_gpu ( df_raw , d = 0.5 , floor = 5e-5 ) # End timer end = time . time () print ( f 'Time { end - start } s' ) # Check print ( 'FD Shape' , df_raw_fd_from_gpu . shape [ 0 ]) print ( 'Correct Shape' , df_raw . shape [ 0 ] - weights . shape [ 0 ]) print ( 'Original Shape' , df_raw . shape [ 0 ]) Time 0.4671785831451416 s FD Shape 999682 Correct Shape 999682 Original Shape 1000000 # Start timer start = time . time () # Create 100m data points df_raw = pd . DataFrame ( np . arange ( large_time_series_length ) * np . random . rand ( large_time_series_length )) df_raw_fd , weights = frac_diff ( df_raw , d = 0.5 , floor = 5e-5 ) # End timer end = time . time () print ( f 'Time { end - start } s' ) # Check print ( 'FD Shape' , df_raw_fd_from_gpu . shape [ 0 ]) print ( 'Correct Shape' , df_raw . shape [ 0 ] - weights . shape [ 0 ]) print ( 'Original Shape' , df_raw . shape [ 0 ]) Time 128.29733324050903 s FD Shape 999682 Correct Shape 999682 Original Shape 1000000","title":"GPU/CPU Fractional Differencing"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#gfd-gpu-fractional-differencing","text":"","title":"GFD: GPU Fractional Differencing"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#summary","text":"Typically we attempt to achieve some form of stationarity via a transformation on our time series through common methods including integer differencing. However, integer differencing unnecessarily removes too much memory to achieve stationarity. An alternative, fractional differencing, allows us to achieve stationarity while maintaining the maximum amount of memory compared to integer differencing. While existing CPU-based implementations are inefficient for running fractional differencing on many large-scale time series, our GPU-based implementation enables rapid fractional differencing. This tutorial walks you through our GPU implementation of fractional differencing (GFD). Also, this tutorial is a special collaboration between industry and academia led by me.","title":"Summary"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#credits","text":"Before we start, I would like to thank and give credits for this particular post to: 1. NVIDIA (Ettikan, Chris and Nick), Amazon AWS, ensemblecap.ai, and NExT++ (NUS School of Computing) 2. Marcos Lopez de Prado for his recent push on the use of fractional differencing of which this guide is based on. 3. Hosking for his paper in 1981 on fractional differencing.","title":"Credits"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#links","text":"Code Repository: https://github.com/ritchieng/fractional_differencing_gpu Presentation: https://www.researchgate.net/publication/335159299_GFD_GPU_Fractional_Differencing_for_Rapid_Large-scale_Stationarizing_of_Time_Series_Data_while_Minimizing_Memory_Loss","title":"Links"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#installation-of-libraries-optional","text":"These steps are for easily running RAPIDS in Google Colab. You can skip this section if you've RAPIDS installed. import pynvml pynvml . nvmlInit () handle = pynvml . nvmlDeviceGetHandleByIndex ( 0 ) device_name = pynvml . nvmlDeviceGetName ( handle ) print ( f 'Currently running on { device_name } ' ) if device_name != b 'Tesla T4' : raise Exception ( \"\"\" Unfortunately this instance does not have a T4 GPU. Please make sure you've configured Colab to request a GPU instance type. Sometimes Colab allocates a Tesla K80 instead of a T4. Resetting the instance. If you get a K80 GPU, try Runtime -> Reset all runtimes -> Keep trying! \"\"\" ) else : print ( 'Woo! You got the right kind of GPU, a Tesla T4!' ) Currently running on b'Tesla T4' Woo! You got the right kind of GPU, a Tesla T4! # intall miniconda ! wget - c https : // repo . continuum . io / miniconda / Miniconda3 - 4.5.4 - Linux - x86_64 . sh ! chmod + x Miniconda3 - 4.5.4 - Linux - x86_64 . sh ! bash ./ Miniconda3 - 4.5.4 - Linux - x86_64 . sh - b - f - p / usr / local # install RAPIDS packages ! conda install - q - y -- prefix / usr / local - c conda - forge \\ - c rapidsai - nightly / label / cuda10 .0 - c nvidia / label / cuda10 .0 \\ cudf cuml # set environment vars import sys , os , shutil sys . path . append ( '/usr/local/lib/python3.6/site-packages/' ) os . environ [ 'NUMBAPRO_NVVM' ] = '/usr/local/cuda/nvvm/lib64/libnvvm.so' os . environ [ 'NUMBAPRO_LIBDEVICE' ] = '/usr/local/cuda/nvvm/libdevice/' # copy .so files to current working dir for fn in [ 'libcudf.so' , 'librmm.so' ]: shutil . copy ( '/usr/local/lib/' + fn , os . getcwd ()) --2019-08-19 05:36:25-- https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh Resolving repo.continuum.io (repo.continuum.io)... 104.18.200.79, 104.18.201.79, 2606:4700::6812:c84f, ... Connecting to repo.continuum.io (repo.continuum.io)|104.18.200.79|:443... connected. HTTP request sent, awaiting response... 416 Requested Range Not Satisfiable The file is already fully retrieved; nothing to do. PREFIX=/usr/local installing: python-3.6.5-hc3d631a_2 ... Python 3.6.5 :: Anaconda, Inc. installing: ca-certificates-2018.03.07-0 ... installing: conda-env-2.6.0-h36134e3_1 ... installing: libgcc-ng-7.2.0-hdf63c60_3 ... installing: libstdcxx-ng-7.2.0-hdf63c60_3 ... installing: libffi-3.2.1-hd88cf55_4 ... installing: ncurses-6.1-hf484d3e_0 ... installing: openssl-1.0.2o-h20670df_0 ... installing: tk-8.6.7-hc745277_3 ... installing: xz-5.2.4-h14c3975_4 ... installing: yaml-0.1.7-had09818_2 ... installing: zlib-1.2.11-ha838bed_2 ... installing: libedit-3.1.20170329-h6b74fdf_2 ... installing: readline-7.0-ha6073c6_4 ... installing: sqlite-3.23.1-he433501_0 ... installing: asn1crypto-0.24.0-py36_0 ... installing: certifi-2018.4.16-py36_0 ... installing: chardet-3.0.4-py36h0f667ec_1 ... installing: idna-2.6-py36h82fb2a8_1 ... installing: pycosat-0.6.3-py36h0a5515d_0 ... installing: pycparser-2.18-py36hf9f622e_1 ... installing: pysocks-1.6.8-py36_0 ... installing: ruamel_yaml-0.15.37-py36h14c3975_2 ... installing: six-1.11.0-py36h372c433_1 ... installing: cffi-1.11.5-py36h9745a5d_0 ... installing: setuptools-39.2.0-py36_0 ... installing: cryptography-2.2.2-py36h14c3975_0 ... installing: wheel-0.31.1-py36_0 ... installing: pip-10.0.1-py36_0 ... installing: pyopenssl-18.0.0-py36_0 ... installing: urllib3-1.22-py36hbe7ace6_0 ... installing: requests-2.18.4-py36he2e5f8d_1 ... installing: conda-4.5.4-py36_0 ... unlinking: ca-certificates-2019.6.16-hecc5488_0 unlinking: certifi-2019.6.16-py36_1 unlinking: conda-4.7.11-py36_0 unlinking: cryptography-2.7-py36h72c5cf5_0 unlinking: libgcc-ng-9.1.0-hdf63c60_0 unlinking: libstdcxx-ng-9.1.0-hdf63c60_0 unlinking: openssl-1.1.1c-h516909a_0 unlinking: python-3.6.7-h381d211_1004 unlinking: sqlite-3.28.0-h8b20d00_0 unlinking: tk-8.6.9-hed695b0_1002 installation finished. WARNING: You currently have a PYTHONPATH environment variable set. This may cause unexpected behavior when running the Python interpreter in Miniconda3. For best results, please verify that your PYTHONPATH only points to directories of packages that are compatible with the Python interpreter in Miniconda3: /usr/local Solving environment: ...working... done ## Package Plan ## environment location: /usr/local added / updated specs: - cudf - cuml The following packages will be UPDATED: ca-certificates: 2018.03.07-0 --> 2019.6.16-hecc5488_0 conda-forge certifi: 2018.4.16-py36_0 --> 2019.6.16-py36_1 conda-forge conda: 4.5.4-py36_0 --> 4.7.11-py36_0 conda-forge cryptography: 2.2.2-py36h14c3975_0 --> 2.7-py36h72c5cf5_0 conda-forge libgcc-ng: 7.2.0-hdf63c60_3 --> 9.1.0-hdf63c60_0 libstdcxx-ng: 7.2.0-hdf63c60_3 --> 9.1.0-hdf63c60_0 openssl: 1.0.2o-h20670df_0 --> 1.1.1c-h516909a_0 conda-forge python: 3.6.5-hc3d631a_2 --> 3.6.7-h381d211_1004 conda-forge sqlite: 3.23.1-he433501_0 --> 3.28.0-h8b20d00_0 conda-forge tk: 8.6.7-hc745277_3 --> 8.6.9-hed695b0_1002 conda-forge Preparing transaction: ...working... done Verifying transaction: ...working... done Executing transaction: ...working... done You would need to do a Runtime > Restart and Run All on your Google Colab notebook at this step.","title":"Installation of Libraries (Optional)"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#imports","text":"# Critical imports for GPU cuDF import nvstrings , nvcategory , cudf # Other imports import numpy as np import pandas as pd import time import pandas_datareader.data as web from datetime import datetime from matplotlib import pyplot as plt from numba import cuda","title":"Imports"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#plotting-style","text":"# Display settings: just to make it slightly prettier # There'll be a separate post on beautiful plots % matplotlib inline figsize = ( 25 , 6 ) plt . style . use ( 'fivethirtyeight' )","title":"Plotting Style"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#necessity-of-fractional-differencing","text":"Before we get into why fractional differencing is critical, let us inspect a simple time series pulled from the Fed's database: S&P 500.","title":"Necessity of Fractional Differencing"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#pulling-processing-plotting-spx-time-series","text":"# Read SPX data 2010-2019 from FED database: https://fred.stlouisfed.org/categories/94 asset_name = 'SPX' start = datetime ( 2010 , 1 , 1 ) end = datetime ( 2019 , 8 , 1 ) df_raw = web . DataReader ( 'SP500' , 'fred' , start , end ) # Basic clean up data: dropna (there are other more robust treatments) df_raw = df_raw . dropna () df_raw . columns = [ asset_name ] df_raw . plot ( figsize = figsize ) plt . title ( 'S&P 500 Absolute Levels' );","title":"Pulling, Processing &amp; Plotting SPX Time Series"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#integer-differencing-1-day-returns-spx-time-series","text":"Traditionally we typically might difference our time series by one day (daily returns) or more to reach some form of stationarity via tests like ADF, KPSS, PP and more. However, these forms of integer differencing causes more information than needed to be lost. # One day returns calculation through differencing by 1 differencing_factor = 1 df_daily_returns = ( df_raw - df_raw . shift ( differencing_factor )) / df_raw . shift ( differencing_factor ) * 100 df_daily_returns . plot ( figsize = figsize ) plt . title ( f ' { asset_name } Daily Returns via Differencing 1 Day' );","title":"Integer Differencing (1 Day Returns) SPX Time Series"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#integer-differencing-causes-unnecessary-memory-loss","text":"Because of this reason, we propose fractional differencing to achieve stationarity while minimizing memory loss.","title":"Integer Differencing Causes Unnecessary Memory Loss"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#basic-fractional-differencing","text":"Fractional differencing allows us to achieve stationarity while maintaining the maximum amount of memory compared to integer differencing. This was originally introduced in 1981 in his paper \u201cFractional Differencing\u201d by J. R. M. Hosking1 and subsequent work by others concentrated on fast and efficient implementations for fractional differentiation for continuous stochastic processes. Recently, fractional differencing was introduced for financial time series through the fixed window fractional differencing instead of the expanding window method by Marcos Lopez de Prado2.","title":"Basic Fractional Differencing"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#fractional-differencing-weight-function-and-plot","text":"Weights formula: \\(w_k = -w_{k-1} \\frac{d - k + 1}{k}\\) Weight converges to zero: \\(w_k \\rightarrow 0\\) def get_weights ( d , num_k ): r \"\"\"Calculate weights ($w$) for each lag ($k$) through $w_k = -w_{k-1} \\frac{d - k + 1}{k}$. Args: d (int): differencing value. num_k (int): number of lags (typically length of timeseries) to calculate w. \"\"\" w_k = np . array ([ 1 ]) for k in range ( 1 , num_k ): w_k = np . append ( w_k , - w_k [ - 1 ] * (( d - k + 1 )) / k ) w_k = w_k . reshape ( - 1 , 1 ) return w_k def plot_weights ( range_d , num_k , num_d_interval ): r \"\"\"Plot weights ($w$) for each lag ($k$) for varying differencing value ($d$). Args: range_d (list): range of differencing values to plot. num_k (int): number of lags (typically length of timeseries) to plot. num_d_interval (int): number of d interval. \"\"\" # Get differencing values interval = np . linspace ( range_d [ 0 ], range_d [ 1 ], num_d_interval ) # Dataframe of lags (rows) x number of differencing intervals (columns) df_wk = pd . DataFrame ( np . zeros (( num_k , num_d_interval ))) # Get weights array per differencing value for i , d in enumerate ( interval ): df_wk [ i ] = get_weights ( d , num_k ) # Rename columns for legend df_wk . columns = [ round ( x , 1 ) for x in interval ] # Plot df_wk . plot ( figsize = figsize ) plt . title ( 'Lag weights ($w_k$) for various differencing values (d)' ) plt . legend ( title = 'Differencing value (d)' ) plt . ylabel ( 'Weights (w)' ) plt . xlabel ( 'Lag (k)' ) plt . show () # Return weights return df_wk df_wk = plot_weights ( range_d = [ 0 , 1 ], num_k = 7 , num_d_interval = 6 )","title":"Fractional Differencing Weight Function and Plot"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#fractional-differencing-weight-function-table","text":"# Dataframe for lag weights (w) for various differencing values (d) df_wk .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } 0.0 0.2 0.4 0.6 0.8 1.0 0 1.0 1.000000 1.000000 1.000000 1.000000 1.0 1 -0.0 -0.200000 -0.400000 -0.600000 -0.800000 -1.0 2 -0.0 -0.080000 -0.120000 -0.120000 -0.080000 0.0 3 -0.0 -0.048000 -0.064000 -0.056000 -0.032000 0.0 4 -0.0 -0.033600 -0.041600 -0.033600 -0.017600 0.0 5 -0.0 -0.025536 -0.029952 -0.022848 -0.011264 0.0 6 -0.0 -0.020429 -0.022963 -0.016755 -0.007885 0.0","title":"Fractional Differencing Weight Function Table"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#fractional-differencing-value-calculation-toy-example","text":"Assume spot have values 100, 99, 98, 97, 96, 95, 94 from k=0 to k=6","title":"Fractional Differencing Value Calculation Toy Example"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#for-d-0-no-differencing","text":"differenced_value = 100 * 1 + 99 * 0 + 98 * 0 + 97 * 0 + 96 * 0 + 95 * 0 + 94 * 0 print ( f 'Differenced value: { differenced_value } ' ) df_wk . iloc [:, 0 ] Differenced value: 100 0 1.0 1 -0.0 2 -0.0 3 -0.0 4 -0.0 5 -0.0 6 -0.0 Name: 0.0, dtype: float64","title":"For \\(d = 0\\) (no differencing)"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#for-d-04-fractional-differencing","text":"differenced_value = 100 * 1 + 99 * - 0.400000 + 98 * - 0.120000 + 97 * - 0.064000 + 96 * - 0.041600 + 95 * - 0.041600 + 94 * - 0.041600 print ( f 'Differenced value: { differenced_value } ' ) df_wk . iloc [:, 1 ] Differenced value: 30.576000000000004 0 1.000000 1 -0.200000 2 -0.080000 3 -0.048000 4 -0.033600 5 -0.025536 6 -0.020429 Name: 0.2, dtype: float64","title":"For \\(d = 0.4\\) (fractional differencing)"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#for-d-1-integer-differencing","text":"differenced_value = 100 * 1 + 99 * - 1 + 98 * 0 + 97 * 0 + 96 * 0 + 95 * 0 + 94 * 0 print ( f 'Differenced value: { differenced_value } ' ) df_wk . iloc [:, - 1 ] Differenced value: 1 0 1.0 1 -1.0 2 0.0 3 0.0 4 0.0 5 0.0 6 0.0 Name: 1.0, dtype: float64","title":"For \\(d = 1\\) (integer differencing)"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#cumulative-sum-of-weights-behavior","text":"Higher differencing value would accelerate cumulative sum weight decay This causes: Less information taken from further lags More information loss from further lags df_wk . cumsum () . plot ( figsize = figsize ) plt . title ( 'Cumulative Sum of Lag weights ($\\sum_0^k w_k$) for various differencing values (d)' ) plt . legend ( title = 'Differencing value ($d$)' ) plt . ylabel ( 'Cumulative Sum of Weights ($\\sum_0^k w_k$)' ) plt . xlabel ( 'Lag (k)' );","title":"Cumulative Sum of Weights Behavior"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#fixed-window-fractional-differencing-cpu","text":"","title":"Fixed Window Fractional Differencing (CPU)"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#floored-weights-function","text":"For computational efficiency, we want to stop the calculation of weights when the weights are too small (below a certain small predetermined small value). Image we had a time series with 1,000,000 data points, we do not want to compute the weights up till 1m lag for our latest data point! We can simply calculate the weights till they're too small which is what this function does. def get_weights_floored ( d , num_k , floor = 1e-3 ): r \"\"\"Calculate weights ($w$) for each lag ($k$) through $w_k = -w_{k-1} \\frac{d - k + 1}{k}$ provided weight above a minimum value (floor) for the weights to prevent computation of weights for the entire time series. Args: d (int): differencing value. num_k (int): number of lags (typically length of timeseries) to calculate w. floor (float): minimum value for the weights for computational efficiency. \"\"\" w_k = np . array ([ 1 ]) k = 1 while k < num_k : w_k_latest = - w_k [ - 1 ] * (( d - k + 1 )) / k if abs ( w_k_latest ) <= floor : break w_k = np . append ( w_k , w_k_latest ) k += 1 w_k = w_k . reshape ( - 1 , 1 ) return w_k # Show large time series being stopped # Notice how max lag is at 9? d = 0.5 num_k = 1000 weights = get_weights_floored ( d = d , num_k = num_k ) pd . DataFrame ( weights ) . plot ( legend = False , figsize = figsize ) # Plot plt . title ( f 'Lag weights ($w_k$) for differencing value $d= { d } $' ) plt . ylabel ( 'Weights (w)' ) plt . xlabel ( 'Lag (k)' );","title":"Floored Weights Function"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#fixed-window-fractional-differencing-function-cpu","text":"Here, we use the new floored weights function compared to our normal weights function Calculate our differenced values through the dot product of our transposed weights matrix and our original time series within that window Keep repeating by shifting the window by one time step until the end of the time series def frac_diff ( df , d , floor = 1e-3 ): r \"\"\"Fractionally difference time series via CPU. Args: df (pd.DataFrame): dataframe of raw time series values. d (float): differencing value from 0 to 1 where > 1 has no FD. floor (float): minimum value of weights, ignoring anything smaller. \"\"\" # Get weights window weights = get_weights_floored ( d = d , num_k = len ( df ), floor = floor ) weights_window_size = len ( weights ) # Reverse weights weights = weights [:: - 1 ] # Blank fractionally differenced series to be filled df_fd = [] # Slide window of time series, to calculated fractionally differenced values # per window for idx in range ( weights_window_size , df . shape [ 0 ]): # Dot product of weights and original values # to get fractionally differenced values date_idx = df . index [ idx ] df_fd . append ( np . dot ( weights . T , df . iloc [ idx - weights_window_size : idx ]) . item ()) # Return FD values and weights df_fd = pd . DataFrame ( df_fd ) return df_fd , weights # Start timer start = time . time () df_raw_fd , weights = frac_diff ( df_raw , d = 0.5 , floor = 5e-5 ) # End timer end = time . time () print ( f 'Time { end - start } s' ) # Plot df_raw_fd . plot ( figsize = figsize ); Time 0.5417799949645996 s import multiprocessing multiprocessing . cpu_count () 4 Existing CPU-based implementations are inefficient for running fractional differencing on many large-scale time-series. GPU-based implementations provide an avenue to adapt to this century\u2019s big data requirements.","title":"Fixed Window Fractional Differencing Function (CPU)"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#fixed-window-fractional-differencing-function-gpu","text":"This gives a slight 1.5x to 2x speed-up on an NVIDIA T4 on Google Colab for this tiny dataset For larger time series in line with real-world situations, it can easily be up to 100x to 1000x dataset (shown below). def moving_dot_product_kernel ( in_data , out , window_size , weights ): # Set the first window_size-1 rows in each chunk to np.nan due # insufficient history for i in range ( cuda . threadIdx . x , window_size - 1 , cuda . blockDim . x ): out [ i ] = np . nan # Compute dot product of preceding window_size rows for i in range ( cuda . threadIdx . x + window_size - 1 , in_data . size , cuda . blockDim . x ): rolling_dot_product = 0.0 k = 0 for j in range ( i - window_size + 1 , i + 1 ): rolling_dot_product += in_data [ j ] * weights [ k ][ 0 ] k += 1 out [ i ] = rolling_dot_product def frac_diff_gpu ( df , d , floor = 1e-3 ): r \"\"\"Fractionally difference time series via GPU. Args: df (pd.DataFrame): dataframe of raw time series values. d (float): differencing value from 0 to 1 where > 1 has no FD. floor (float): minimum value of weights, ignoring anything smaller. \"\"\" # Bring dataframe to GPU, reset index for GPU dot product kernel gdf_raw = cudf . from_pandas ( df ) . reset_index ( drop = True ) gdf_raw . columns = [ 'in_data' ] # Get weights window weights = get_weights_floored ( d = d , num_k = len ( gdf_raw ), floor = floor ) weights_window_size = len ( weights ) # Reverse weights and as contiguous weights = np . ascontiguousarray ( weights [:: - 1 ]) # Bring weights to GPU gdf_weights = cudf . DataFrame () gdf_weights [ gdf_raw . columns [ 0 ]] = weights . reshape ( - 1 ) # Length of data data_length = len ( gdf_raw ) # T4: max of 518 threads per block. # V100: max 1024 threads per block threads_per_block = 518 # Chunk size split # This has to be improved, but as a v0.1, it's sufficient to show speed-up # Up to easily 100 million data points trunk_size = data_length # Get fractionally differenced time series through GPU function gdf_raw_fd = gdf_raw . apply_chunks ( moving_dot_product_kernel , incols = [ 'in_data' ], outcols = dict ( out = np . float64 ), kwargs = dict ( window_size = weights_window_size , weights = weights ), chunks = list ( range ( 0 , data_length , trunk_size )) + [ data_length ], tpb = threads_per_block ) # Bring to CPU for normal manipulation df_raw_fd = gdf_raw_fd . to_pandas () . dropna () . iloc [: - 1 , 1 ] return df_raw_fd , weights # Start timer start = time . time () df_raw_fd_from_gpu , weights = frac_diff_gpu ( df_raw , d = 0.5 , floor = 5e-5 ) # End timer end = time . time () print ( f 'Time { end - start } s' ) # Plot df_raw_fd_from_gpu . plot ( figsize = figsize ); Time 0.27262187004089355 s","title":"Fixed Window Fractional Differencing Function (GPU)"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#check-values","text":"# Compare tail values print ( 'Tail values check' ) print ( df_raw_fd_from_gpu . tail () . values ) print ( df_raw_fd . tail () . values . reshape ( - 1 ,)) Tail values check [107.00340988 133.85520208 117.92878691 109.44132697 79.32562638] [107.00340988 133.85520208 117.92878691 109.44132697 79.32562638] # Compare tail values print ( 'Head values check' ) print ( df_raw_fd_from_gpu . head () . values ) print ( df_raw_fd . head () . values . reshape ( - 1 ,)) Head values check [56.74989213 52.56766288 47.32421832 45.89772154 37.74401501] [56.74989213 52.56766288 47.32421832 45.89772154 37.74401501] num_rows_true = ( df_raw_fd . values . astype ( np . float32 ) == df_raw_fd_from_gpu . values . astype ( np . float32 )) . astype ( int ) . sum () total_rows = df_raw_fd . shape [ 0 ] print ( f 'Number of rows equal: { num_rows_true } ' ) print ( f 'Total number of rows: { total_rows } ' ) Number of rows equal: 2093 Total number of rows: 2093","title":"Check Values"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#stationarity","text":"\"A stationary time series is one whose statistical properties such as mean, variance, autocorrelation, etc. are all constant over time\", Robert Nau, Duke University Essentially what we are trying to do here with fractional differencing is to attempt to have a stationary time series without losing too much memory. There are many ways to check if a time series is stationary, but we will be using 1 test here (they're more, but it's not the purpose of this tutorial) Augmented Dickey\u2013Fuller (ADF) test: check for unit root","title":"Stationarity"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#adf-test","text":"# Import adf/kpss from statsmodels.tsa.stattools import adfuller import warnings warnings . filterwarnings ( \"ignore\" ) # Ignore deprecations for cleaner output","title":"ADF Test"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#adf-raw-data-d0","text":"# ADF on raw data print ( 'Stationarity Test for SPX Absolute Levels (d=0)' ) print ( '-' * 50 ) result = adfuller ( df_raw [ 'SPX' ], regression = 'c' ) print ( f 't-stat \\n\\t { result [ 0 ] : .2f } ' ) print ( f 'p-value \\n\\t { result [ 1 ] : .2f } ' ) print ( f 'Critical Values' ) for key , value in result [ 4 ] . items (): print ( f ' \\t { key } : { value : .2f } ' ) # test statistic > test statistic (1%) AND p-value > 0.01 # Fail to reject the null hypothesis that there's unit root at the 1% significance level # ==> Fail to reject H0 ==> Unit root ==> Non-stationary Stationarity Test for SPX Absolute Levels (d=0) -------------------------------------------------- t-stat -0.11 p-value 0.95 Critical Values 1%: -3.43 5%: -2.86 10%: -2.57","title":"ADF: raw data (d=0)"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#adf-daily-return-data-d1","text":"# ADF on 1 day returns (d=1) print ( 'Stationarity Test for SPX Daily Returns (d=1)' ) print ( '-' * 50 ) result = adfuller ( df_raw [ 'SPX' ] . pct_change ( 1 ) . dropna (), regression = 'c' ) print ( f 't-stat \\n\\t { result [ 0 ] : .2f } ' ) print ( f 'p-value \\n\\t { result [ 1 ] : .2f } ' ) print ( f 'Critical Values' ) for key , value in result [ 4 ] . items (): print ( f ' \\t { key } : { value : .2f } ' ) # test statistic < test statistic (1%) AND p-value < 0.01 # Reject the null hypothesis that there's unit root at the 1% significance level # ==> Reject H0 ==> No unit root ==> Stationary Stationarity Test for SPX Daily Returns (d=1) -------------------------------------------------- t-stat -11.12 p-value 0.00 Critical Values 1%: -3.43 5%: -2.86 10%: -2.57","title":"ADF: daily return data (d=1)"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#adf-fractionally-differenced-data-d05","text":"# ADF on fractionally differenced values print ( 'Stationarity Test (ADF) for SPX Fractionally Differenced (d=0.5)' ) print ( '-' * 50 ) df_raw_fd_from_gpu , weights = frac_diff_gpu ( df_raw , d = 0.5 , floor = 5e-5 ) result = adfuller ( df_raw_fd_from_gpu , regression = 'c' ) print ( f 't-stat \\n\\t { result [ 0 ] : .2f } ' ) print ( f 'p-value \\n\\t { result [ 1 ] : .2f } ' ) print ( f 'Critical Values' ) for key , value in result [ 4 ] . items (): print ( f ' \\t { key } : { value : .2f } ' ) # test statistic < test statistic (1%) AND p-value < 0.01 # Reject the null hypothesis that there's unit root at the 1% significance level # ==> Reject H0 ==> No unit root ==> Stationary Stationarity Test (ADF) for SPX Fractionally Differenced (d=0.5) -------------------------------------------------- t-stat -3.86 p-value 0.00 Critical Values 1%: -3.43 5%: -2.86 10%: -2.57","title":"ADF: fractionally differenced data (d=0.5)"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#large-scale-rapid-fractional-differencing","text":"Here we test on a 100k, 1m, 10m and 100m datapoint dataframe. # Create 100m data points large_time_series_length = int ( 1e6 ) print ( large_time_series_length ) 1000000 # Start timer start = time . time () df_raw = pd . DataFrame ( np . arange ( large_time_series_length ) * np . random . rand ( large_time_series_length )) df_raw_fd_from_gpu , weights = frac_diff_gpu ( df_raw , d = 0.5 , floor = 5e-5 ) # End timer end = time . time () print ( f 'Time { end - start } s' ) # Check print ( 'FD Shape' , df_raw_fd_from_gpu . shape [ 0 ]) print ( 'Correct Shape' , df_raw . shape [ 0 ] - weights . shape [ 0 ]) print ( 'Original Shape' , df_raw . shape [ 0 ]) Time 0.4671785831451416 s FD Shape 999682 Correct Shape 999682 Original Shape 1000000 # Start timer start = time . time () # Create 100m data points df_raw = pd . DataFrame ( np . arange ( large_time_series_length ) * np . random . rand ( large_time_series_length )) df_raw_fd , weights = frac_diff ( df_raw , d = 0.5 , floor = 5e-5 ) # End timer end = time . time () print ( f 'Time { end - start } s' ) # Check print ( 'FD Shape' , df_raw_fd_from_gpu . shape [ 0 ]) print ( 'Correct Shape' , df_raw . shape [ 0 ] - weights . shape [ 0 ]) print ( 'Original Shape' , df_raw . shape [ 0 ]) Time 128.29733324050903 s FD Shape 999682 Correct Shape 999682 Original Shape 1000000","title":"Large-scale Rapid Fractional Differencing"},{"location":"machine_learning/gpu/rapids_cudf/","text":"RAPIDS cuDF \u00b6 Environment Setup \u00b6 Check Version \u00b6 Python Version \u00b6 # Check Python Version ! python -- version Python 3.8.16 Ubuntu Version \u00b6 # Check Ubuntu Version ! lsb_release - a No LSB modules are available. Distributor ID: Ubuntu Description: Ubuntu 18.04.6 LTS Release: 18.04 Codename: bionic Check CUDA Version \u00b6 # Check CUDA/cuDNN Version ! nvcc - V && which nvcc nvcc: NVIDIA (R) Cuda compiler driver Copyright (c) 2005-2021 NVIDIA Corporation Built on Sun_Feb_14_21:12:58_PST_2021 Cuda compilation tools, release 11.2, V11.2.152 Build cuda_11.2.r11.2/compiler.29618528_0 /usr/local/cuda/bin/nvcc Check GPU Version \u00b6 # Check GPU ! nvidia - smi Wed Jan 4 19:14:22 2023 +-----------------------------------------------------------------------------+ | NVIDIA-SMI 460.32.03 Driver Version: 460.32.03 CUDA Version: 11.2 | |-------------------------------+----------------------+----------------------+ | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | | | | MIG M. | |===============================+======================+======================| | 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 | | N/A 48C P0 29W / 70W | 0MiB / 15109MiB | 0% Default | | | | N/A | +-------------------------------+----------------------+----------------------+ +-----------------------------------------------------------------------------+ | Processes: | | GPU GI CI PID Type Process name GPU Memory | | ID ID Usage | |=============================================================================| | No running processes found | +-----------------------------------------------------------------------------+ Setup: \u00b6 This set up script: Checks to make sure that the GPU is RAPIDS compatible Installs the current stable version of RAPIDSAI's core libraries using pip, which are: cuDF cuML cuGraph xgboost This will complete in about 3-4 minutes Please use the RAPIDS Conda Colab Template notebook if you need to install any of RAPIDS Extended libraries, such as: - cuSpatial - cuSignal - cuxFilter - cuCIM OR - nightly versions of any library # This get the RAPIDS-Colab install files and test check your GPU. Run this and the next cell only. # Please read the output of this cell. If your Colab Instance is not RAPIDS compatible, it will warn you and give you remediation steps. ! git clone https : // github . com / rapidsai / rapidsai - csp - utils . git ! python rapidsai - csp - utils / colab / pip - install . py Cloning into 'rapidsai-csp-utils'... remote: Enumerating objects: 328, done.\u001b[K remote: Counting objects: 100% (157/157), done.\u001b[K remote: Compressing objects: 100% (102/102), done.\u001b[K remote: Total 328 (delta 92), reused 98 (delta 55), pack-reused 171\u001b[K Receiving objects: 100% (328/328), 94.64 KiB | 18.93 MiB/s, done. Resolving deltas: 100% (154/154), done. Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/ Collecting pynvml Downloading pynvml-11.4.1-py3-none-any.whl (46 kB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 47.0/47.0 KB 6.1 MB/s eta 0:00:00 Installing collected packages: pynvml Successfully installed pynvml-11.4.1 *********************************************************************** Woo! Your instance has the right kind of GPU, a Tesla T4! We will now install RAPIDS via pip! Please stand by, should be quick... *********************************************************************** Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/, https://pypi.ngc.nvidia.com Collecting cudf-cu11 Downloading https://developer.download.nvidia.com/compute/redist/cudf-cu11/cudf_cu11-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (442.8 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 442.8/442.8 MB 3.5 MB/s eta 0:00:00 Collecting cuml-cu11 Downloading https://developer.download.nvidia.com/compute/redist/cuml-cu11/cuml_cu11-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1359.8 MB) tcmalloc: large alloc 1359798272 bytes == 0x3116000 @ 0x7f53812b21e7 0x4d30a0 0x4d312c 0x5d6f4c 0x51edd1 0x51ef5b 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x5d8868 0x4997a2 0x55cd91 0x5d8941 0x49abe4 0x55cd91 0x5d8941 0x4997a2 tcmalloc: large alloc 1699749888 bytes == 0x541e4000 @ 0x7f53812b3615 0x5d6f4c 0x51edd1 0x51ef5b 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x5d8868 0x4997a2 0x55cd91 0x5d8941 0x49abe4 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 tcmalloc: large alloc 1359798272 bytes == 0x3116000 @ 0x7f53812b21e7 0x4d30a0 0x5dede2 0x6758aa 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4fe318 0x5da092 0x62042c 0x5d8d8c 0x561f80 0x4fd2db 0x4997c7 0x4fd8b5 0x4997c7 0x4fd8b5 0x49abe4 0x4f5fe9 0x55e146 0x4f5fe9 0x55e146 0x4f5fe9 0x55e146 0x5d8868 0x5da092 0x587116 \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.4/1.4 GB 1.3 MB/s eta 0:00:00 Collecting cugraph-cu11 Downloading https://developer.download.nvidia.com/compute/redist/cugraph-cu11/cugraph_cu11-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1028.4 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.0/1.0 GB 1.9 MB/s eta 0:00:00 Requirement already satisfied: numba>=0.56.2 in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (0.56.4) Requirement already satisfied: numpy in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (1.21.6) Collecting ptxcompiler-cu11 Downloading https://developer.download.nvidia.com/compute/redist/ptxcompiler-cu11/ptxcompiler_cu11-0.7.0.post1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.8 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 8.8/8.8 MB 99.1 MB/s eta 0:00:00 Collecting cuda-python<12.0,>=11.7.1 Downloading cuda_python-11.8.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (16.2 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 16.2/16.2 MB 77.6 MB/s eta 0:00:00 Requirement already satisfied: pyarrow==9.0.0 in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (9.0.0) Requirement already satisfied: pandas<1.6.0dev0,>=1.0 in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (1.3.5) Requirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (4.4.0) Requirement already satisfied: cupy-cuda11x in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (11.0.0) Requirement already satisfied: cachetools in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (5.2.0) Requirement already satisfied: fsspec>=0.6.0 in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (2022.11.0) Collecting protobuf<3.21.0a0,>=3.20.1 Downloading protobuf-3.20.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl (1.0 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.0/1.0 MB 46.4 MB/s eta 0:00:00 Collecting rmm-cu11 Downloading https://developer.download.nvidia.com/compute/redist/rmm-cu11/rmm_cu11-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.8 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.8/1.8 MB 61.1 MB/s eta 0:00:00 Collecting cubinlinker-cu11 Downloading https://developer.download.nvidia.com/compute/redist/cubinlinker-cu11/cubinlinker_cu11-0.3.0.post1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.8 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 8.8/8.8 MB 99.9 MB/s eta 0:00:00 Requirement already satisfied: packaging in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (21.3) Collecting nvtx>=0.2.1 Downloading nvtx-0.2.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (453 kB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 453.6/453.6 KB 28.4 MB/s eta 0:00:00 Requirement already satisfied: seaborn in /usr/local/lib/python3.8/dist-packages (from cuml-cu11) (0.11.2) Collecting raft-dask-cu11 Downloading https://developer.download.nvidia.com/compute/redist/raft-dask-cu11/raft_dask_cu11-22.12.0.post1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (210.5 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 210.5/210.5 MB 6.7 MB/s eta 0:00:00 Requirement already satisfied: scipy in /usr/local/lib/python3.8/dist-packages (from cuml-cu11) (1.7.3) Collecting treelite==3.0.1 Downloading treelite-3.0.1-py3-none-manylinux2014_x86_64.whl (864 kB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 864.6/864.6 KB 38.0 MB/s eta 0:00:00 Collecting treelite-runtime==3.0.1 Downloading treelite_runtime-3.0.1-py3-none-manylinux2014_x86_64.whl (191 kB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 191.9/191.9 KB 25.3 MB/s eta 0:00:00 Collecting dask-cudf-cu11 Downloading https://developer.download.nvidia.com/compute/redist/dask-cudf-cu11/dask_cudf_cu11-22.12.0.post1-py3-none-any.whl (76 kB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 76.6/76.6 KB 12.2 MB/s eta 0:00:00 Collecting pylibraft-cu11 Downloading https://developer.download.nvidia.com/compute/redist/pylibraft-cu11/pylibraft_cu11-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (580.3 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 580.3/580.3 MB 3.2 MB/s eta 0:00:00 Collecting pylibcugraph-cu11 Downloading https://developer.download.nvidia.com/compute/redist/pylibcugraph-cu11/pylibcugraph_cu11-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1627.2 MB) tcmalloc: large alloc 1627185152 bytes == 0x541e8000 @ 0x7f53812b21e7 0x4d30a0 0x4d312c 0x5d6f4c 0x51edd1 0x51ef5b 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x5d8868 0x4997a2 0x55cd91 0x5d8941 0x49abe4 0x55cd91 0x5d8941 0x4997a2 tcmalloc: large alloc 2033983488 bytes == 0xb51b6000 @ 0x7f53812b3615 0x5d6f4c 0x51edd1 0x51ef5b 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x5d8868 0x4997a2 0x55cd91 0x5d8941 0x49abe4 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.6/1.6 GB 1.1 MB/s eta 0:00:00 Collecting dask-cuda Downloading dask_cuda-22.12.0-py3-none-any.whl (121 kB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 121.1/121.1 KB 17.5 MB/s eta 0:00:00 Requirement already satisfied: cython in /usr/local/lib/python3.8/dist-packages (from cuda-python<12.0,>=11.7.1->cudf-cu11) (0.29.32) Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.8/dist-packages (from numba>=0.56.2->cudf-cu11) (5.2.0) Requirement already satisfied: llvmlite<0.40,>=0.39.0dev0 in /usr/local/lib/python3.8/dist-packages (from numba>=0.56.2->cudf-cu11) (0.39.1) Requirement already satisfied: setuptools in /usr/local/lib/python3.8/dist-packages (from numba>=0.56.2->cudf-cu11) (57.4.0) Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.8/dist-packages (from pandas<1.6.0dev0,>=1.0->cudf-cu11) (2.8.2) Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.8/dist-packages (from pandas<1.6.0dev0,>=1.0->cudf-cu11) (2022.7) Requirement already satisfied: fastrlock>=0.5 in /usr/local/lib/python3.8/dist-packages (from cupy-cuda11x->cudf-cu11) (0.8.1) Requirement already satisfied: zict>=0.1.3 in /usr/local/lib/python3.8/dist-packages (from dask-cuda->cugraph-cu11) (2.2.0) Collecting distributed==2022.11.1 Downloading distributed-2022.11.1-py3-none-any.whl (923 kB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 923.4/923.4 KB 50.7 MB/s eta 0:00:00 Collecting dask==2022.11.1 Downloading dask-2022.11.1-py3-none-any.whl (1.1 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.1/1.1 MB 52.6 MB/s eta 0:00:00 Requirement already satisfied: pynvml>=11.0.0 in /usr/local/lib/python3.8/dist-packages (from dask-cuda->cugraph-cu11) (11.4.1) Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.8/dist-packages (from dask==2022.11.1->dask-cuda->cugraph-cu11) (7.1.2) Requirement already satisfied: toolz>=0.8.2 in /usr/local/lib/python3.8/dist-packages (from dask==2022.11.1->dask-cuda->cugraph-cu11) (0.12.0) Requirement already satisfied: partd>=0.3.10 in /usr/local/lib/python3.8/dist-packages (from dask==2022.11.1->dask-cuda->cugraph-cu11) (1.3.0) Requirement already satisfied: cloudpickle>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from dask==2022.11.1->dask-cuda->cugraph-cu11) (1.5.0) Requirement already satisfied: pyyaml>=5.3.1 in /usr/local/lib/python3.8/dist-packages (from dask==2022.11.1->dask-cuda->cugraph-cu11) (6.0) Requirement already satisfied: tornado<6.2,>=6.0.3 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1->dask-cuda->cugraph-cu11) (6.0.4) Requirement already satisfied: locket>=1.0.0 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1->dask-cuda->cugraph-cu11) (1.0.0) Requirement already satisfied: tblib>=1.6.0 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1->dask-cuda->cugraph-cu11) (1.7.0) Requirement already satisfied: jinja2 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1->dask-cuda->cugraph-cu11) (2.11.3) Requirement already satisfied: psutil>=5.0 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1->dask-cuda->cugraph-cu11) (5.4.8) Requirement already satisfied: urllib3 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1->dask-cuda->cugraph-cu11) (1.24.3) Requirement already satisfied: sortedcontainers!=2.0.0,!=2.0.1 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1->dask-cuda->cugraph-cu11) (2.4.0) Requirement already satisfied: msgpack>=0.6.0 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1->dask-cuda->cugraph-cu11) (1.0.4) Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging->cudf-cu11) (3.0.9) Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.8/dist-packages (from raft-dask-cu11->cuml-cu11) (1.2.0) Collecting ucx-py-cu11 Downloading https://developer.download.nvidia.com/compute/redist/ucx-py-cu11/ucx_py_cu11-0.29.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.3 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 8.3/8.3 MB 72.8 MB/s eta 0:00:00 Requirement already satisfied: matplotlib>=2.2 in /usr/local/lib/python3.8/dist-packages (from seaborn->cuml-cu11) (3.2.2) Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.8/dist-packages (from matplotlib>=2.2->seaborn->cuml-cu11) (0.11.0) Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.8/dist-packages (from matplotlib>=2.2->seaborn->cuml-cu11) (1.4.4) Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.8/dist-packages (from python-dateutil>=2.7.3->pandas<1.6.0dev0,>=1.0->cudf-cu11) (1.15.0) Requirement already satisfied: heapdict in /usr/local/lib/python3.8/dist-packages (from zict>=0.1.3->dask-cuda->cugraph-cu11) (1.0.1) Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.8/dist-packages (from importlib-metadata->numba>=0.56.2->cudf-cu11) (3.11.0) Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.8/dist-packages (from jinja2->distributed==2022.11.1->dask-cuda->cugraph-cu11) (2.0.1) Installing collected packages: ptxcompiler-cu11, nvtx, cubinlinker-cu11, ucx-py-cu11, protobuf, cuda-python, treelite-runtime, treelite, dask, rmm-cu11, distributed, pylibraft-cu11, dask-cuda, cudf-cu11, raft-dask-cu11, pylibcugraph-cu11, dask-cudf-cu11, cuml-cu11, cugraph-cu11 Attempting uninstall: protobuf Found existing installation: protobuf 3.19.6 Uninstalling protobuf-3.19.6: Successfully uninstalled protobuf-3.19.6 Attempting uninstall: dask Found existing installation: dask 2022.2.1 Uninstalling dask-2022.2.1: Successfully uninstalled dask-2022.2.1 Attempting uninstall: distributed Found existing installation: distributed 2022.2.1 Uninstalling distributed-2022.2.1: Successfully uninstalled distributed-2022.2.1 ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. tensorflow 2.9.2 requires protobuf<3.20,>=3.9.2, but you have protobuf 3.20.3 which is incompatible. tensorboard 2.9.1 requires protobuf<3.20,>=3.9.2, but you have protobuf 3.20.3 which is incompatible. Successfully installed cubinlinker-cu11-0.3.0.post1 cuda-python-11.8.1 cudf-cu11-22.12.0 cugraph-cu11-22.12.0 cuml-cu11-22.12.0 dask-2022.11.1 dask-cuda-22.12.0 dask-cudf-cu11-22.12.0.post1 distributed-2022.11.1 nvtx-0.2.5 protobuf-3.20.3 ptxcompiler-cu11-0.7.0.post1 pylibcugraph-cu11-22.12.0 pylibraft-cu11-22.12.0 raft-dask-cu11-22.12.0.post1 rmm-cu11-22.12.0 treelite-3.0.1 treelite-runtime-3.0.1 ucx-py-cu11-0.29.0 Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/ Collecting cupy-cuda11x Downloading cupy_cuda11x-11.4.0-cp38-cp38-manylinux1_x86_64.whl (93.7 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 93.7/93.7 MB 10.7 MB/s eta 0:00:00 Requirement already satisfied: numpy<1.26,>=1.20 in /usr/local/lib/python3.8/dist-packages (from cupy-cuda11x) (1.21.6) Requirement already satisfied: fastrlock>=0.5 in /usr/local/lib/python3.8/dist-packages (from cupy-cuda11x) (0.8.1) Installing collected packages: cupy-cuda11x Successfully installed cupy-cuda11x-11.4.0 *********************************************************************** With the new pip install complete, please do not run any further installation commands from the conda based installation methods!!! In your personal files, you can delete these cells. RAPIDSAI owned templates/notebooks should already be updated with no action needed. *********************************************************************** Critical Imports \u00b6 # Critical imports import cudf import cuml import os import numpy as np import pandas as pd Creating \u00b6 Create a Series of integers \u00b6 gdf = cudf . Series ([ 1 , 2 , 3 , 4 , 5 , 6 ]) print ( gdf ) print ( type ( gdf )) 0 1 1 2 2 3 3 4 4 5 5 6 dtype: int64 <class 'cudf.core.series.Series'> Create a Series of floats \u00b6 gdf = cudf . Series ([ 1. , 2. , 3. , 4. , 5. , 6. ]) print ( gdf ) 0 1.0 1 2.0 2 3.0 3 4.0 4 5.0 5 6.0 dtype: float64 Create a Series of strings \u00b6 gdf = cudf . Series ([ 'a' , 'b' , 'c' ]) print ( gdf ) 0 a 1 b 2 c dtype: object Create 3 column DataFrame \u00b6 Consisting of dates, integers and floats # Import import datetime as dt # Using a dictionary of key-value pairs # Each key in the dictionary represents a category # The key is the category's name # The value is a list of the values in that category gdf = cudf . DataFrame ({ # Create 10 busindates ess from 1st January 2019 via pandas 'dates' : pd . date_range ( '1/1/2019' , periods = 10 , freq = 'B' ), # Integers 'integers' : [ i for i in range ( 10 )], # Floats 'floats' : [ float ( i ) for i in range ( 10 )] }) # Print dataframe print ( gdf ) dates integers floats 0 2019-01-01 0 0.0 1 2019-01-02 1 1.0 2 2019-01-03 2 2.0 3 2019-01-04 3 3.0 4 2019-01-07 4 4.0 5 2019-01-08 5 5.0 6 2019-01-09 6 6.0 7 2019-01-10 7 7.0 8 2019-01-11 8 8.0 9 2019-01-14 9 9.0 Create 2 column Dataframe \u00b6 Consisting of integers and string category # Using a dictionary # Each key in the dictionary represents a category # The key is the category's name # The value is a list of the values in that category gdf = cudf . DataFrame ({ 'integers' : [ 1 , 2 , 3 , 4 ], 'string' : [ 'a' , 'b' , 'c' , 'd' ] }) print ( gdf ) integers string 0 1 a 1 2 b 2 3 c 3 4 d Create a 2 Column Dataframe with Pandas Bridge \u00b6 Consisting of integers and string category For all string columns, you must convert them to type category for filtering functions to work intuitively (for now) # Create pandas dataframe pandas_df = pd . DataFrame ({ 'integers' : [ 1 , 2 , 3 , 4 ], 'strings' : [ 'a' , 'b' , 'c' , 'd' ] }) # Convert string column to category format pandas_df [ 'strings' ] = pandas_df [ 'strings' ] . astype ( 'category' ) # Bridge from pandas to cudf gdf = cudf . DataFrame . from_pandas ( pandas_df ) # Print dataframe print ( gdf ) integers strings 0 1 a 1 2 b 2 3 c 3 4 d Viewing \u00b6 Printing Column Names \u00b6 gdf . columns Index(['integers', 'strings'], dtype='object') Viewing Top of DataFrame \u00b6 num_of_rows_to_view = 2 print ( gdf . head ( num_of_rows_to_view )) integers strings 0 1 a 1 2 b Viewing Bottom of DataFrame \u00b6 num_of_rows_to_view = 3 print ( gdf . tail ( num_of_rows_to_view )) integers strings 1 2 b 2 3 c 3 4 d Filtering \u00b6 Method 1: Query \u00b6 Filtering Integers/Floats by Column Values \u00b6 This only works for floats and integers, not for strings # DO NOT RUN # TOFIX: `cffi` package version mismatch error print ( gdf . query ( 'integers == 1' )) integers strings 0 1 a Filtering Strings by Column Values \u00b6 This only works for floats and integers, not for strings so this will return an error! print ( gdf . query ( 'strings == a' )) --------------------------------------------------------------------------- KeyError Traceback (most recent call last) /usr/local/lib/python3.8/dist-packages/cudf/core/dataframe.py in extract_col(df, col) 7558 try: -> 7559 return df._data[col] 7560 except KeyError: /usr/local/lib/python3.8/dist-packages/cudf/core/column_accessor.py in __getitem__(self, key) 154 def __getitem__(self, key: Any) -> ColumnBase: --> 155 return self._data[key] 156 KeyError: 'a' During handling of the above exception, another exception occurred: KeyError Traceback (most recent call last) <ipython-input-17-5cfd0345d51c> in <module> ----> 1 print(gdf.query('strings == a')) /usr/local/lib/python3.8/dist-packages/cudf/core/dataframe.py in query(self, expr, local_dict) 4172 } 4173 # Run query -> 4174 boolmask = queryutils.query_execute(self, expr, callenv) 4175 return self._apply_boolean_mask(boolmask) 4176 /usr/local/lib/python3.8/dist-packages/cudf/utils/queryutils.py in query_execute(df, expr, callenv) 212 213 # prepare col args --> 214 colarrays = [cudf.core.dataframe.extract_col(df, col) for col in columns] 215 216 # wait to check the types until we know which cols are used /usr/local/lib/python3.8/dist-packages/cudf/utils/queryutils.py in <listcomp>(.0) 212 213 # prepare col args --> 214 colarrays = [cudf.core.dataframe.extract_col(df, col) for col in columns] 215 216 # wait to check the types until we know which cols are used /usr/local/lib/python3.8/dist-packages/cudf/core/dataframe.py in extract_col(df, col) 7565 ): 7566 return df.index._data.columns[0] -> 7567 return df.index._data[col] 7568 7569 /usr/local/lib/python3.8/dist-packages/cudf/core/column_accessor.py in __getitem__(self, key) 153 154 def __getitem__(self, key: Any) -> ColumnBase: --> 155 return self._data[key] 156 157 def __setitem__(self, key: Any, value: Any): KeyError: 'a' Method 2: Simple Columns \u00b6 Filtering Strings by Column Values \u00b6 # Filtering based on the string column print ( gdf [ gdf . strings == 'b' ]) integers strings 1 2 b Filtering Integers/Floats by Column Values \u00b6 # Filtering based on the string column print ( gdf [ gdf . integers == 2 ]) integers strings 1 2 b Method 2: Simple Rows \u00b6 Filtering by Row Numbers \u00b6 # Filter rows 0 to 2 (not inclusive of the third row with the index 2) print ( gdf [ 0 : 2 ]) integers strings 0 1 a 1 2 b Method 3: loc[rows, columns] \u00b6 # The syntax is as follows loc[rows, columns] allowing you to choose rows and columns accordingly # The example allows us to filter the first 3 rows (inclusive) of the column integers print ( gdf . loc [ 0 : 2 , [ 'integers' ]]) integers 0 1 1 2 2 3","title":"GPU DataFrames"},{"location":"machine_learning/gpu/rapids_cudf/#rapids-cudf","text":"","title":"RAPIDS cuDF"},{"location":"machine_learning/gpu/rapids_cudf/#environment-setup","text":"","title":"Environment Setup"},{"location":"machine_learning/gpu/rapids_cudf/#check-version","text":"","title":"Check Version"},{"location":"machine_learning/gpu/rapids_cudf/#python-version","text":"# Check Python Version ! python -- version Python 3.8.16","title":"Python Version"},{"location":"machine_learning/gpu/rapids_cudf/#ubuntu-version","text":"# Check Ubuntu Version ! lsb_release - a No LSB modules are available. Distributor ID: Ubuntu Description: Ubuntu 18.04.6 LTS Release: 18.04 Codename: bionic","title":"Ubuntu Version"},{"location":"machine_learning/gpu/rapids_cudf/#check-cuda-version","text":"# Check CUDA/cuDNN Version ! nvcc - V && which nvcc nvcc: NVIDIA (R) Cuda compiler driver Copyright (c) 2005-2021 NVIDIA Corporation Built on Sun_Feb_14_21:12:58_PST_2021 Cuda compilation tools, release 11.2, V11.2.152 Build cuda_11.2.r11.2/compiler.29618528_0 /usr/local/cuda/bin/nvcc","title":"Check CUDA Version"},{"location":"machine_learning/gpu/rapids_cudf/#check-gpu-version","text":"# Check GPU ! nvidia - smi Wed Jan 4 19:14:22 2023 +-----------------------------------------------------------------------------+ | NVIDIA-SMI 460.32.03 Driver Version: 460.32.03 CUDA Version: 11.2 | |-------------------------------+----------------------+----------------------+ | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | | | | MIG M. | |===============================+======================+======================| | 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 | | N/A 48C P0 29W / 70W | 0MiB / 15109MiB | 0% Default | | | | N/A | +-------------------------------+----------------------+----------------------+ +-----------------------------------------------------------------------------+ | Processes: | | GPU GI CI PID Type Process name GPU Memory | | ID ID Usage | |=============================================================================| | No running processes found | +-----------------------------------------------------------------------------+","title":"Check GPU Version"},{"location":"machine_learning/gpu/rapids_cudf/#setup","text":"This set up script: Checks to make sure that the GPU is RAPIDS compatible Installs the current stable version of RAPIDSAI's core libraries using pip, which are: cuDF cuML cuGraph xgboost This will complete in about 3-4 minutes Please use the RAPIDS Conda Colab Template notebook if you need to install any of RAPIDS Extended libraries, such as: - cuSpatial - cuSignal - cuxFilter - cuCIM OR - nightly versions of any library # This get the RAPIDS-Colab install files and test check your GPU. Run this and the next cell only. # Please read the output of this cell. If your Colab Instance is not RAPIDS compatible, it will warn you and give you remediation steps. ! git clone https : // github . com / rapidsai / rapidsai - csp - utils . git ! python rapidsai - csp - utils / colab / pip - install . py Cloning into 'rapidsai-csp-utils'... remote: Enumerating objects: 328, done.\u001b[K remote: Counting objects: 100% (157/157), done.\u001b[K remote: Compressing objects: 100% (102/102), done.\u001b[K remote: Total 328 (delta 92), reused 98 (delta 55), pack-reused 171\u001b[K Receiving objects: 100% (328/328), 94.64 KiB | 18.93 MiB/s, done. Resolving deltas: 100% (154/154), done. Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/ Collecting pynvml Downloading pynvml-11.4.1-py3-none-any.whl (46 kB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 47.0/47.0 KB 6.1 MB/s eta 0:00:00 Installing collected packages: pynvml Successfully installed pynvml-11.4.1 *********************************************************************** Woo! Your instance has the right kind of GPU, a Tesla T4! We will now install RAPIDS via pip! Please stand by, should be quick... *********************************************************************** Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/, https://pypi.ngc.nvidia.com Collecting cudf-cu11 Downloading https://developer.download.nvidia.com/compute/redist/cudf-cu11/cudf_cu11-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (442.8 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 442.8/442.8 MB 3.5 MB/s eta 0:00:00 Collecting cuml-cu11 Downloading https://developer.download.nvidia.com/compute/redist/cuml-cu11/cuml_cu11-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1359.8 MB) tcmalloc: large alloc 1359798272 bytes == 0x3116000 @ 0x7f53812b21e7 0x4d30a0 0x4d312c 0x5d6f4c 0x51edd1 0x51ef5b 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x5d8868 0x4997a2 0x55cd91 0x5d8941 0x49abe4 0x55cd91 0x5d8941 0x4997a2 tcmalloc: large alloc 1699749888 bytes == 0x541e4000 @ 0x7f53812b3615 0x5d6f4c 0x51edd1 0x51ef5b 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x5d8868 0x4997a2 0x55cd91 0x5d8941 0x49abe4 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 tcmalloc: large alloc 1359798272 bytes == 0x3116000 @ 0x7f53812b21e7 0x4d30a0 0x5dede2 0x6758aa 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4fe318 0x5da092 0x62042c 0x5d8d8c 0x561f80 0x4fd2db 0x4997c7 0x4fd8b5 0x4997c7 0x4fd8b5 0x49abe4 0x4f5fe9 0x55e146 0x4f5fe9 0x55e146 0x4f5fe9 0x55e146 0x5d8868 0x5da092 0x587116 \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.4/1.4 GB 1.3 MB/s eta 0:00:00 Collecting cugraph-cu11 Downloading https://developer.download.nvidia.com/compute/redist/cugraph-cu11/cugraph_cu11-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1028.4 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.0/1.0 GB 1.9 MB/s eta 0:00:00 Requirement already satisfied: numba>=0.56.2 in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (0.56.4) Requirement already satisfied: numpy in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (1.21.6) Collecting ptxcompiler-cu11 Downloading https://developer.download.nvidia.com/compute/redist/ptxcompiler-cu11/ptxcompiler_cu11-0.7.0.post1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.8 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 8.8/8.8 MB 99.1 MB/s eta 0:00:00 Collecting cuda-python<12.0,>=11.7.1 Downloading cuda_python-11.8.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (16.2 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 16.2/16.2 MB 77.6 MB/s eta 0:00:00 Requirement already satisfied: pyarrow==9.0.0 in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (9.0.0) Requirement already satisfied: pandas<1.6.0dev0,>=1.0 in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (1.3.5) Requirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (4.4.0) Requirement already satisfied: cupy-cuda11x in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (11.0.0) Requirement already satisfied: cachetools in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (5.2.0) Requirement already satisfied: fsspec>=0.6.0 in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (2022.11.0) Collecting protobuf<3.21.0a0,>=3.20.1 Downloading protobuf-3.20.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl (1.0 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.0/1.0 MB 46.4 MB/s eta 0:00:00 Collecting rmm-cu11 Downloading https://developer.download.nvidia.com/compute/redist/rmm-cu11/rmm_cu11-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.8 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.8/1.8 MB 61.1 MB/s eta 0:00:00 Collecting cubinlinker-cu11 Downloading https://developer.download.nvidia.com/compute/redist/cubinlinker-cu11/cubinlinker_cu11-0.3.0.post1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.8 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 8.8/8.8 MB 99.9 MB/s eta 0:00:00 Requirement already satisfied: packaging in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (21.3) Collecting nvtx>=0.2.1 Downloading nvtx-0.2.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (453 kB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 453.6/453.6 KB 28.4 MB/s eta 0:00:00 Requirement already satisfied: seaborn in /usr/local/lib/python3.8/dist-packages (from cuml-cu11) (0.11.2) Collecting raft-dask-cu11 Downloading https://developer.download.nvidia.com/compute/redist/raft-dask-cu11/raft_dask_cu11-22.12.0.post1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (210.5 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 210.5/210.5 MB 6.7 MB/s eta 0:00:00 Requirement already satisfied: scipy in /usr/local/lib/python3.8/dist-packages (from cuml-cu11) (1.7.3) Collecting treelite==3.0.1 Downloading treelite-3.0.1-py3-none-manylinux2014_x86_64.whl (864 kB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 864.6/864.6 KB 38.0 MB/s eta 0:00:00 Collecting treelite-runtime==3.0.1 Downloading treelite_runtime-3.0.1-py3-none-manylinux2014_x86_64.whl (191 kB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 191.9/191.9 KB 25.3 MB/s eta 0:00:00 Collecting dask-cudf-cu11 Downloading https://developer.download.nvidia.com/compute/redist/dask-cudf-cu11/dask_cudf_cu11-22.12.0.post1-py3-none-any.whl (76 kB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 76.6/76.6 KB 12.2 MB/s eta 0:00:00 Collecting pylibraft-cu11 Downloading https://developer.download.nvidia.com/compute/redist/pylibraft-cu11/pylibraft_cu11-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (580.3 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 580.3/580.3 MB 3.2 MB/s eta 0:00:00 Collecting pylibcugraph-cu11 Downloading https://developer.download.nvidia.com/compute/redist/pylibcugraph-cu11/pylibcugraph_cu11-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1627.2 MB) tcmalloc: large alloc 1627185152 bytes == 0x541e8000 @ 0x7f53812b21e7 0x4d30a0 0x4d312c 0x5d6f4c 0x51edd1 0x51ef5b 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x5d8868 0x4997a2 0x55cd91 0x5d8941 0x49abe4 0x55cd91 0x5d8941 0x4997a2 tcmalloc: large alloc 2033983488 bytes == 0xb51b6000 @ 0x7f53812b3615 0x5d6f4c 0x51edd1 0x51ef5b 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x5d8868 0x4997a2 0x55cd91 0x5d8941 0x49abe4 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.6/1.6 GB 1.1 MB/s eta 0:00:00 Collecting dask-cuda Downloading dask_cuda-22.12.0-py3-none-any.whl (121 kB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 121.1/121.1 KB 17.5 MB/s eta 0:00:00 Requirement already satisfied: cython in /usr/local/lib/python3.8/dist-packages (from cuda-python<12.0,>=11.7.1->cudf-cu11) (0.29.32) Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.8/dist-packages (from numba>=0.56.2->cudf-cu11) (5.2.0) Requirement already satisfied: llvmlite<0.40,>=0.39.0dev0 in /usr/local/lib/python3.8/dist-packages (from numba>=0.56.2->cudf-cu11) (0.39.1) Requirement already satisfied: setuptools in /usr/local/lib/python3.8/dist-packages (from numba>=0.56.2->cudf-cu11) (57.4.0) Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.8/dist-packages (from pandas<1.6.0dev0,>=1.0->cudf-cu11) (2.8.2) Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.8/dist-packages (from pandas<1.6.0dev0,>=1.0->cudf-cu11) (2022.7) Requirement already satisfied: fastrlock>=0.5 in /usr/local/lib/python3.8/dist-packages (from cupy-cuda11x->cudf-cu11) (0.8.1) Requirement already satisfied: zict>=0.1.3 in /usr/local/lib/python3.8/dist-packages (from dask-cuda->cugraph-cu11) (2.2.0) Collecting distributed==2022.11.1 Downloading distributed-2022.11.1-py3-none-any.whl (923 kB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 923.4/923.4 KB 50.7 MB/s eta 0:00:00 Collecting dask==2022.11.1 Downloading dask-2022.11.1-py3-none-any.whl (1.1 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.1/1.1 MB 52.6 MB/s eta 0:00:00 Requirement already satisfied: pynvml>=11.0.0 in /usr/local/lib/python3.8/dist-packages (from dask-cuda->cugraph-cu11) (11.4.1) Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.8/dist-packages (from dask==2022.11.1->dask-cuda->cugraph-cu11) (7.1.2) Requirement already satisfied: toolz>=0.8.2 in /usr/local/lib/python3.8/dist-packages (from dask==2022.11.1->dask-cuda->cugraph-cu11) (0.12.0) Requirement already satisfied: partd>=0.3.10 in /usr/local/lib/python3.8/dist-packages (from dask==2022.11.1->dask-cuda->cugraph-cu11) (1.3.0) Requirement already satisfied: cloudpickle>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from dask==2022.11.1->dask-cuda->cugraph-cu11) (1.5.0) Requirement already satisfied: pyyaml>=5.3.1 in /usr/local/lib/python3.8/dist-packages (from dask==2022.11.1->dask-cuda->cugraph-cu11) (6.0) Requirement already satisfied: tornado<6.2,>=6.0.3 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1->dask-cuda->cugraph-cu11) (6.0.4) Requirement already satisfied: locket>=1.0.0 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1->dask-cuda->cugraph-cu11) (1.0.0) Requirement already satisfied: tblib>=1.6.0 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1->dask-cuda->cugraph-cu11) (1.7.0) Requirement already satisfied: jinja2 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1->dask-cuda->cugraph-cu11) (2.11.3) Requirement already satisfied: psutil>=5.0 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1->dask-cuda->cugraph-cu11) (5.4.8) Requirement already satisfied: urllib3 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1->dask-cuda->cugraph-cu11) (1.24.3) Requirement already satisfied: sortedcontainers!=2.0.0,!=2.0.1 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1->dask-cuda->cugraph-cu11) (2.4.0) Requirement already satisfied: msgpack>=0.6.0 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1->dask-cuda->cugraph-cu11) (1.0.4) Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging->cudf-cu11) (3.0.9) Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.8/dist-packages (from raft-dask-cu11->cuml-cu11) (1.2.0) Collecting ucx-py-cu11 Downloading https://developer.download.nvidia.com/compute/redist/ucx-py-cu11/ucx_py_cu11-0.29.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.3 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 8.3/8.3 MB 72.8 MB/s eta 0:00:00 Requirement already satisfied: matplotlib>=2.2 in /usr/local/lib/python3.8/dist-packages (from seaborn->cuml-cu11) (3.2.2) Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.8/dist-packages (from matplotlib>=2.2->seaborn->cuml-cu11) (0.11.0) Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.8/dist-packages (from matplotlib>=2.2->seaborn->cuml-cu11) (1.4.4) Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.8/dist-packages (from python-dateutil>=2.7.3->pandas<1.6.0dev0,>=1.0->cudf-cu11) (1.15.0) Requirement already satisfied: heapdict in /usr/local/lib/python3.8/dist-packages (from zict>=0.1.3->dask-cuda->cugraph-cu11) (1.0.1) Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.8/dist-packages (from importlib-metadata->numba>=0.56.2->cudf-cu11) (3.11.0) Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.8/dist-packages (from jinja2->distributed==2022.11.1->dask-cuda->cugraph-cu11) (2.0.1) Installing collected packages: ptxcompiler-cu11, nvtx, cubinlinker-cu11, ucx-py-cu11, protobuf, cuda-python, treelite-runtime, treelite, dask, rmm-cu11, distributed, pylibraft-cu11, dask-cuda, cudf-cu11, raft-dask-cu11, pylibcugraph-cu11, dask-cudf-cu11, cuml-cu11, cugraph-cu11 Attempting uninstall: protobuf Found existing installation: protobuf 3.19.6 Uninstalling protobuf-3.19.6: Successfully uninstalled protobuf-3.19.6 Attempting uninstall: dask Found existing installation: dask 2022.2.1 Uninstalling dask-2022.2.1: Successfully uninstalled dask-2022.2.1 Attempting uninstall: distributed Found existing installation: distributed 2022.2.1 Uninstalling distributed-2022.2.1: Successfully uninstalled distributed-2022.2.1 ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. tensorflow 2.9.2 requires protobuf<3.20,>=3.9.2, but you have protobuf 3.20.3 which is incompatible. tensorboard 2.9.1 requires protobuf<3.20,>=3.9.2, but you have protobuf 3.20.3 which is incompatible. Successfully installed cubinlinker-cu11-0.3.0.post1 cuda-python-11.8.1 cudf-cu11-22.12.0 cugraph-cu11-22.12.0 cuml-cu11-22.12.0 dask-2022.11.1 dask-cuda-22.12.0 dask-cudf-cu11-22.12.0.post1 distributed-2022.11.1 nvtx-0.2.5 protobuf-3.20.3 ptxcompiler-cu11-0.7.0.post1 pylibcugraph-cu11-22.12.0 pylibraft-cu11-22.12.0 raft-dask-cu11-22.12.0.post1 rmm-cu11-22.12.0 treelite-3.0.1 treelite-runtime-3.0.1 ucx-py-cu11-0.29.0 Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/ Collecting cupy-cuda11x Downloading cupy_cuda11x-11.4.0-cp38-cp38-manylinux1_x86_64.whl (93.7 MB) \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 93.7/93.7 MB 10.7 MB/s eta 0:00:00 Requirement already satisfied: numpy<1.26,>=1.20 in /usr/local/lib/python3.8/dist-packages (from cupy-cuda11x) (1.21.6) Requirement already satisfied: fastrlock>=0.5 in /usr/local/lib/python3.8/dist-packages (from cupy-cuda11x) (0.8.1) Installing collected packages: cupy-cuda11x Successfully installed cupy-cuda11x-11.4.0 *********************************************************************** With the new pip install complete, please do not run any further installation commands from the conda based installation methods!!! In your personal files, you can delete these cells. RAPIDSAI owned templates/notebooks should already be updated with no action needed. ***********************************************************************","title":"Setup:"},{"location":"machine_learning/gpu/rapids_cudf/#critical-imports","text":"# Critical imports import cudf import cuml import os import numpy as np import pandas as pd","title":"Critical Imports"},{"location":"machine_learning/gpu/rapids_cudf/#creating","text":"","title":"Creating"},{"location":"machine_learning/gpu/rapids_cudf/#create-a-series-of-integers","text":"gdf = cudf . Series ([ 1 , 2 , 3 , 4 , 5 , 6 ]) print ( gdf ) print ( type ( gdf )) 0 1 1 2 2 3 3 4 4 5 5 6 dtype: int64 <class 'cudf.core.series.Series'>","title":"Create a Series of integers"},{"location":"machine_learning/gpu/rapids_cudf/#create-a-series-of-floats","text":"gdf = cudf . Series ([ 1. , 2. , 3. , 4. , 5. , 6. ]) print ( gdf ) 0 1.0 1 2.0 2 3.0 3 4.0 4 5.0 5 6.0 dtype: float64","title":"Create a Series of floats"},{"location":"machine_learning/gpu/rapids_cudf/#create-a-series-of-strings","text":"gdf = cudf . Series ([ 'a' , 'b' , 'c' ]) print ( gdf ) 0 a 1 b 2 c dtype: object","title":"Create a  Series of strings"},{"location":"machine_learning/gpu/rapids_cudf/#create-3-column-dataframe","text":"Consisting of dates, integers and floats # Import import datetime as dt # Using a dictionary of key-value pairs # Each key in the dictionary represents a category # The key is the category's name # The value is a list of the values in that category gdf = cudf . DataFrame ({ # Create 10 busindates ess from 1st January 2019 via pandas 'dates' : pd . date_range ( '1/1/2019' , periods = 10 , freq = 'B' ), # Integers 'integers' : [ i for i in range ( 10 )], # Floats 'floats' : [ float ( i ) for i in range ( 10 )] }) # Print dataframe print ( gdf ) dates integers floats 0 2019-01-01 0 0.0 1 2019-01-02 1 1.0 2 2019-01-03 2 2.0 3 2019-01-04 3 3.0 4 2019-01-07 4 4.0 5 2019-01-08 5 5.0 6 2019-01-09 6 6.0 7 2019-01-10 7 7.0 8 2019-01-11 8 8.0 9 2019-01-14 9 9.0","title":"Create 3 column DataFrame"},{"location":"machine_learning/gpu/rapids_cudf/#create-2-column-dataframe","text":"Consisting of integers and string category # Using a dictionary # Each key in the dictionary represents a category # The key is the category's name # The value is a list of the values in that category gdf = cudf . DataFrame ({ 'integers' : [ 1 , 2 , 3 , 4 ], 'string' : [ 'a' , 'b' , 'c' , 'd' ] }) print ( gdf ) integers string 0 1 a 1 2 b 2 3 c 3 4 d","title":"Create 2 column Dataframe"},{"location":"machine_learning/gpu/rapids_cudf/#create-a-2-column-dataframe-with-pandas-bridge","text":"Consisting of integers and string category For all string columns, you must convert them to type category for filtering functions to work intuitively (for now) # Create pandas dataframe pandas_df = pd . DataFrame ({ 'integers' : [ 1 , 2 , 3 , 4 ], 'strings' : [ 'a' , 'b' , 'c' , 'd' ] }) # Convert string column to category format pandas_df [ 'strings' ] = pandas_df [ 'strings' ] . astype ( 'category' ) # Bridge from pandas to cudf gdf = cudf . DataFrame . from_pandas ( pandas_df ) # Print dataframe print ( gdf ) integers strings 0 1 a 1 2 b 2 3 c 3 4 d","title":"Create a 2 Column  Dataframe with Pandas Bridge"},{"location":"machine_learning/gpu/rapids_cudf/#viewing","text":"","title":"Viewing"},{"location":"machine_learning/gpu/rapids_cudf/#printing-column-names","text":"gdf . columns Index(['integers', 'strings'], dtype='object')","title":"Printing Column Names"},{"location":"machine_learning/gpu/rapids_cudf/#viewing-top-of-dataframe","text":"num_of_rows_to_view = 2 print ( gdf . head ( num_of_rows_to_view )) integers strings 0 1 a 1 2 b","title":"Viewing Top of DataFrame"},{"location":"machine_learning/gpu/rapids_cudf/#viewing-bottom-of-dataframe","text":"num_of_rows_to_view = 3 print ( gdf . tail ( num_of_rows_to_view )) integers strings 1 2 b 2 3 c 3 4 d","title":"Viewing Bottom of DataFrame"},{"location":"machine_learning/gpu/rapids_cudf/#filtering","text":"","title":"Filtering"},{"location":"machine_learning/gpu/rapids_cudf/#method-1-query","text":"","title":"Method 1: Query"},{"location":"machine_learning/gpu/rapids_cudf/#filtering-integersfloats-by-column-values","text":"This only works for floats and integers, not for strings # DO NOT RUN # TOFIX: `cffi` package version mismatch error print ( gdf . query ( 'integers == 1' )) integers strings 0 1 a","title":"Filtering Integers/Floats by Column Values"},{"location":"machine_learning/gpu/rapids_cudf/#filtering-strings-by-column-values","text":"This only works for floats and integers, not for strings so this will return an error! print ( gdf . query ( 'strings == a' )) --------------------------------------------------------------------------- KeyError Traceback (most recent call last) /usr/local/lib/python3.8/dist-packages/cudf/core/dataframe.py in extract_col(df, col) 7558 try: -> 7559 return df._data[col] 7560 except KeyError: /usr/local/lib/python3.8/dist-packages/cudf/core/column_accessor.py in __getitem__(self, key) 154 def __getitem__(self, key: Any) -> ColumnBase: --> 155 return self._data[key] 156 KeyError: 'a' During handling of the above exception, another exception occurred: KeyError Traceback (most recent call last) <ipython-input-17-5cfd0345d51c> in <module> ----> 1 print(gdf.query('strings == a')) /usr/local/lib/python3.8/dist-packages/cudf/core/dataframe.py in query(self, expr, local_dict) 4172 } 4173 # Run query -> 4174 boolmask = queryutils.query_execute(self, expr, callenv) 4175 return self._apply_boolean_mask(boolmask) 4176 /usr/local/lib/python3.8/dist-packages/cudf/utils/queryutils.py in query_execute(df, expr, callenv) 212 213 # prepare col args --> 214 colarrays = [cudf.core.dataframe.extract_col(df, col) for col in columns] 215 216 # wait to check the types until we know which cols are used /usr/local/lib/python3.8/dist-packages/cudf/utils/queryutils.py in <listcomp>(.0) 212 213 # prepare col args --> 214 colarrays = [cudf.core.dataframe.extract_col(df, col) for col in columns] 215 216 # wait to check the types until we know which cols are used /usr/local/lib/python3.8/dist-packages/cudf/core/dataframe.py in extract_col(df, col) 7565 ): 7566 return df.index._data.columns[0] -> 7567 return df.index._data[col] 7568 7569 /usr/local/lib/python3.8/dist-packages/cudf/core/column_accessor.py in __getitem__(self, key) 153 154 def __getitem__(self, key: Any) -> ColumnBase: --> 155 return self._data[key] 156 157 def __setitem__(self, key: Any, value: Any): KeyError: 'a'","title":"Filtering Strings by Column Values"},{"location":"machine_learning/gpu/rapids_cudf/#method-2-simple-columns","text":"","title":"Method 2:  Simple Columns"},{"location":"machine_learning/gpu/rapids_cudf/#filtering-strings-by-column-values_1","text":"# Filtering based on the string column print ( gdf [ gdf . strings == 'b' ]) integers strings 1 2 b","title":"Filtering Strings by Column Values"},{"location":"machine_learning/gpu/rapids_cudf/#filtering-integersfloats-by-column-values_1","text":"# Filtering based on the string column print ( gdf [ gdf . integers == 2 ]) integers strings 1 2 b","title":"Filtering Integers/Floats by Column Values"},{"location":"machine_learning/gpu/rapids_cudf/#method-2-simple-rows","text":"","title":"Method 2:  Simple Rows"},{"location":"machine_learning/gpu/rapids_cudf/#filtering-by-row-numbers","text":"# Filter rows 0 to 2 (not inclusive of the third row with the index 2) print ( gdf [ 0 : 2 ]) integers strings 0 1 a 1 2 b","title":"Filtering by Row Numbers"},{"location":"machine_learning/gpu/rapids_cudf/#method-3-locrows-columns","text":"# The syntax is as follows loc[rows, columns] allowing you to choose rows and columns accordingly # The example allows us to filter the first 3 rows (inclusive) of the column integers print ( gdf . loc [ 0 : 2 , [ 'integers' ]]) integers 0 1 1 2 2 3","title":"Method 3:  loc[rows, columns]"},{"location":"news/ammi_facebook_google_recap_2018_11_21/","text":"AMMI (AIMS) Recap Supported by Facebook and Google \u00b6 To the future of AI in and beyond Africa \u00b6 Helped to teach these students from The African Masters of Machine Intelligence (AMMI) by The African Institute for Mathematical Sciences (AIMS) with Alfredo Canziani. A unique pedagogy is required for deep learning beginning with intuition, culminating in just enough math, and ending with programming. This approached has worked when I delivered my deep learning course to over 3000 students over 120 countries. And I continue to champion this pedagogy. I\u2019m very pleased that I share the same passion with Alfedo when he delivered his interactive lectures. Importantly, I see great potential when I personally interacted with students here. I see future deep learning researchers and applied machine intelligence experts who have the potential to make groundbreaking changes in the fields of healthcare, agriculture and education throughout Africa. With the right education, mentorship and opportunities, they would be pioneers. Simply put, deep learning talent is underexplored in this continent. I am thankful to Alfredo Canziani who has been my best pal and inspirational guy while I was there, Moustapha Cisse for creating these opportunities for young African students to thrive, #Facebook and #Google for supporting this program, and Pedro Antonio Mart\u00ednez Mediano and Marc Deisenroff. Coincidentally, I would also like to thank Yoshua Bengio for holding ICLR in Africa in 2020 for the first time that he announced recently. Finally, really appreciate how #PyTorch has made programming neural networks more approachable and it was made possible with the amazing PyTorch community sparked by Soumith Chintala. Speaking about community, I got to contribute to the deep learning community with the help from supportive groups and individuals from the global deep learning community. Likewise, we should continually help these young Africans like any other. To the future of AI in and beyond Africa. Looking forward to help Moustapha in any way I can to nurture AI talent \ud83d\ude42 I'm being a fanboy here, but Yann Lecun liked my post on Facebook ! Cheers, Ritchie","title":"AMMI (AIMS) supported by Facebook and Google, November 2018"},{"location":"news/ammi_facebook_google_recap_2018_11_21/#ammi-aims-recap-supported-by-facebook-and-google","text":"","title":"AMMI (AIMS) Recap Supported by Facebook and Google"},{"location":"news/ammi_facebook_google_recap_2018_11_21/#to-the-future-of-ai-in-and-beyond-africa","text":"Helped to teach these students from The African Masters of Machine Intelligence (AMMI) by The African Institute for Mathematical Sciences (AIMS) with Alfredo Canziani. A unique pedagogy is required for deep learning beginning with intuition, culminating in just enough math, and ending with programming. This approached has worked when I delivered my deep learning course to over 3000 students over 120 countries. And I continue to champion this pedagogy. I\u2019m very pleased that I share the same passion with Alfedo when he delivered his interactive lectures. Importantly, I see great potential when I personally interacted with students here. I see future deep learning researchers and applied machine intelligence experts who have the potential to make groundbreaking changes in the fields of healthcare, agriculture and education throughout Africa. With the right education, mentorship and opportunities, they would be pioneers. Simply put, deep learning talent is underexplored in this continent. I am thankful to Alfredo Canziani who has been my best pal and inspirational guy while I was there, Moustapha Cisse for creating these opportunities for young African students to thrive, #Facebook and #Google for supporting this program, and Pedro Antonio Mart\u00ednez Mediano and Marc Deisenroff. Coincidentally, I would also like to thank Yoshua Bengio for holding ICLR in Africa in 2020 for the first time that he announced recently. Finally, really appreciate how #PyTorch has made programming neural networks more approachable and it was made possible with the amazing PyTorch community sparked by Soumith Chintala. Speaking about community, I got to contribute to the deep learning community with the help from supportive groups and individuals from the global deep learning community. Likewise, we should continually help these young Africans like any other. To the future of AI in and beyond Africa. Looking forward to help Moustapha in any way I can to nurture AI talent \ud83d\ude42 I'm being a fanboy here, but Yann Lecun liked my post on Facebook ! Cheers, Ritchie","title":"To the future of AI in and beyond Africa"},{"location":"news/dbs_gpu_rapids_nvidia_ensemble_frac_diff/","text":"GPU Fractional Differencing with NVIDIA RAPIDS \u00b6 Introduction to Deep Learning \u00b6 I created and led a workshop on accelerating fractional differencing with NVIDIA's RAPIDS in DBS. This is our article on medium with NVIDIA on our work. You can check out our presentation and open-source code on Github. Thanks to our industry partners DBS, NVIDIA and ensemblecap.ai Cheers, Ritchie","title":"Fractional Differencing with GPU (GFD), DBS and NVIDIA, September 2019"},{"location":"news/dbs_gpu_rapids_nvidia_ensemble_frac_diff/#gpu-fractional-differencing-with-nvidia-rapids","text":"","title":"GPU Fractional Differencing with NVIDIA RAPIDS"},{"location":"news/dbs_gpu_rapids_nvidia_ensemble_frac_diff/#introduction-to-deep-learning","text":"I created and led a workshop on accelerating fractional differencing with NVIDIA's RAPIDS in DBS. This is our article on medium with NVIDIA on our work. You can check out our presentation and open-source code on Github. Thanks to our industry partners DBS, NVIDIA and ensemblecap.ai Cheers, Ritchie","title":"Introduction to Deep Learning"},{"location":"news/deep_learning_wizard_1y_2018_06_01/","text":"Featured on PyTorch Website \u00b6 PyTorch a Year Later \u00b6 We are featured on PyTorch website's post I used PyTorch from day 1 and I fell in love with it. One year later, PyTorch's community has grown tremendously. I thank Facebook AI Research (FAIR) and everyone who contributed to building one of the best deep learning framework. In particular Soumith Chintala from FAIR where he is continually fixing bugs and making PyTorch better everyday. A year later, Deep Learning Wizard has taught more than 2000 students across 100+ countries, and we continue to strive to release new and updated content to enable anyone to build customized deep learning systems to solve many problems. And I'm trying to push for PyTorch workshops at NVIDIA Deep Learning Institute in Singapore and across the world, so stay tuned! A big shoutout for Alfredo Canziani who's currently a postdoc at NYU co-teaching with Yann LeCun on deep learning classes and he's releasing another technical course on PyTorch that is going to be awesome. To more great years ahead for PyTorch Cheers, Ritchie Ng","title":"Featured on PyTorch Website 2018"},{"location":"news/deep_learning_wizard_1y_2018_06_01/#featured-on-pytorch-website","text":"","title":"Featured on PyTorch Website"},{"location":"news/deep_learning_wizard_1y_2018_06_01/#pytorch-a-year-later","text":"We are featured on PyTorch website's post I used PyTorch from day 1 and I fell in love with it. One year later, PyTorch's community has grown tremendously. I thank Facebook AI Research (FAIR) and everyone who contributed to building one of the best deep learning framework. In particular Soumith Chintala from FAIR where he is continually fixing bugs and making PyTorch better everyday. A year later, Deep Learning Wizard has taught more than 2000 students across 100+ countries, and we continue to strive to release new and updated content to enable anyone to build customized deep learning systems to solve many problems. And I'm trying to push for PyTorch workshops at NVIDIA Deep Learning Institute in Singapore and across the world, so stay tuned! A big shoutout for Alfredo Canziani who's currently a postdoc at NYU co-teaching with Yann LeCun on deep learning classes and he's releasing another technical course on PyTorch that is going to be awesome. To more great years ahead for PyTorch Cheers, Ritchie Ng","title":"PyTorch a Year Later"},{"location":"news/deep_learning_wizard_nvidia_inception_2018_05_01/","text":"We Are an NVIDIA Inception Partner \u00b6 We did it! \u00b6 After almost a year, we are an NVIDIA Inception Partner now! \"NVIDIA Inception Partner Deep Learning Wizard is proudly an NVIDIA Inception Partner, empowering people across the world to leverage on deep learning via open-source programming languages and frameworks to solve problems. Cheers, Ritchie Ng","title":"NVIDIA Inception Partner Status, Singapore, May 2017"},{"location":"news/deep_learning_wizard_nvidia_inception_2018_05_01/#we-are-an-nvidia-inception-partner","text":"","title":"We Are an NVIDIA Inception Partner"},{"location":"news/deep_learning_wizard_nvidia_inception_2018_05_01/#we-did-it","text":"After almost a year, we are an NVIDIA Inception Partner now! \"NVIDIA Inception Partner Deep Learning Wizard is proudly an NVIDIA Inception Partner, empowering people across the world to leverage on deep learning via open-source programming languages and frameworks to solve problems. Cheers, Ritchie Ng","title":"We did it!"},{"location":"news/defence_and_science_technology_agency_dsta_nvidia_talk_2016_06/","text":"Deep Learning Demystified, Defence and Science Technology Agency (DSTA) \u00b6 Introduction to Deep Learning \u00b6 I gave an introductory talk on deep learning to more than 1000 students at BrainHack held by DSTA in June 2019 . I'm really excited for Singapore's progress in leveraging on deep learning for the defense sector. There are a lot of applications and areas of research. Special thanks to DSTA, NVIDIA and ensemblecap.ai Cheers, Ritchie","title":"Deep Learning Introduction, Defence and Science Technology Agency (DSTA) and NVIDIA, June 2019"},{"location":"news/defence_and_science_technology_agency_dsta_nvidia_talk_2016_06/#deep-learning-demystified-defence-and-science-technology-agency-dsta","text":"","title":"Deep Learning Demystified, Defence and Science Technology Agency (DSTA)"},{"location":"news/defence_and_science_technology_agency_dsta_nvidia_talk_2016_06/#introduction-to-deep-learning","text":"I gave an introductory talk on deep learning to more than 1000 students at BrainHack held by DSTA in June 2019 . I'm really excited for Singapore's progress in leveraging on deep learning for the defense sector. There are a lot of applications and areas of research. Special thanks to DSTA, NVIDIA and ensemblecap.ai Cheers, Ritchie","title":"Introduction to Deep Learning"},{"location":"news/detect_waterbone_debris_ai_for_social_good_icml_2019_06/","text":"Oral Presentation for AI for Social Good Workshop ICML \u00b6 Detecting Waterborne Debris with Sim2Real and Randomization \u00b6 We'll be presenting our work in ICML this June. Special thanks to everyone working on this project: Jie Fu (MILA, Polytechnic Montr\u00e9al); Ritchie Ng (National University of Singapore); Mirgahney Mohamed (The African Institute for Mathematical Sciences); Yi Tay (Nanyang Technological University); Kris Sankaran (Montreal Institute for Learning Algorithms); Shangbang Long (Peking University); Alfredo Canziani (New York University); Chris J Pal (MILA, Polytechnique Montr\u00e9al, Element AI); Moustapha Cisse (Google Research) Cheers, Ritchie","title":"Oral Presentation for AI for Social Good Workshop ICML, June 2019"},{"location":"news/detect_waterbone_debris_ai_for_social_good_icml_2019_06/#oral-presentation-for-ai-for-social-good-workshop-icml","text":"","title":"Oral Presentation for AI for Social Good Workshop ICML"},{"location":"news/detect_waterbone_debris_ai_for_social_good_icml_2019_06/#detecting-waterborne-debris-with-sim2real-and-randomization","text":"We'll be presenting our work in ICML this June. Special thanks to everyone working on this project: Jie Fu (MILA, Polytechnic Montr\u00e9al); Ritchie Ng (National University of Singapore); Mirgahney Mohamed (The African Institute for Mathematical Sciences); Yi Tay (Nanyang Technological University); Kris Sankaran (Montreal Institute for Learning Algorithms); Shangbang Long (Peking University); Alfredo Canziani (New York University); Chris J Pal (MILA, Polytechnique Montr\u00e9al, Element AI); Moustapha Cisse (Google Research) Cheers, Ritchie","title":"Detecting Waterborne Debris with Sim2Real and Randomization"},{"location":"news/facebook_pytorch_devcon_recap_2018_10_02/","text":"Recap of Facebook PyTorch Devcon \u00b6 Woo! \u00b6 Proud of how far PyTorch has come. Thanks Soumith Chintala, it is a great journey and it just started. And thanks to Adam Paszke too! Finally got to catch up with Soumith Chintala, Alfredo Canziani and Marek Bardo\u0144ski and found Andrej Karpathy! Back to Singapore tonight \ud83e\udd17 Here are some photos of my trip! Cheers, Ritchie","title":"Recap of Facebook PyTorch Developer Conference, San Francisco, September 2018"},{"location":"news/facebook_pytorch_devcon_recap_2018_10_02/#recap-of-facebook-pytorch-devcon","text":"","title":"Recap of Facebook PyTorch Devcon"},{"location":"news/facebook_pytorch_devcon_recap_2018_10_02/#woo","text":"Proud of how far PyTorch has come. Thanks Soumith Chintala, it is a great journey and it just started. And thanks to Adam Paszke too! Finally got to catch up with Soumith Chintala, Alfredo Canziani and Marek Bardo\u0144ski and found Andrej Karpathy! Back to Singapore tonight \ud83e\udd17 Here are some photos of my trip! Cheers, Ritchie","title":"Woo!"},{"location":"news/facebook_pytorch_developer_conference_2018_09_05/","text":"Facebook PyTorch Developer Conference \u00b6 We are heading down! \u00b6 In barely 2 short years, PyTorch (Facebook) will be hosting their first PyTorch Developer Conference in San Francisco, USA. I will be heading down thanks to Soumith Chintala for the invite and arrangements. Looking forward to meet anyone there. The PyTorch ecosystem has grown tremendously from when I first started using it. To this date, I've taught more than 3000 students worldwide in 120+ countries and every single wizard has fallen in love with it! Cheers, Ritchie Ng","title":"Facebook PyTorch Developer Conference, San Francisco, September 2018"},{"location":"news/facebook_pytorch_developer_conference_2018_09_05/#facebook-pytorch-developer-conference","text":"","title":"Facebook PyTorch Developer Conference"},{"location":"news/facebook_pytorch_developer_conference_2018_09_05/#we-are-heading-down","text":"In barely 2 short years, PyTorch (Facebook) will be hosting their first PyTorch Developer Conference in San Francisco, USA. I will be heading down thanks to Soumith Chintala for the invite and arrangements. Looking forward to meet anyone there. The PyTorch ecosystem has grown tremendously from when I first started using it. To this date, I've taught more than 3000 students worldwide in 120+ countries and every single wizard has fallen in love with it! Cheers, Ritchie Ng","title":"We are heading down!"},{"location":"news/it_youth_leader_2019_03_11/","text":"IT Youth Leader of the Year 2019 Award \u00b6 We got it! \u00b6 I am very happy to be the IT Youth Leader of the Year 2019 in Singapore for my industry and non-profit contribution in AI at Deep Learning Wizard , ensemblecap.ai, NUS, and NVIDIA! I would like to thank the professor who changed my life, believed in me and nominated me for consideration, Professor Chua Tat-Seng from NExT Centre, NUS School of Computing. Big thanks to everyone who helped for this in NExT Centre particularly Choon Meng, Tek Min, Chua Yi Hao, and Eugene Tan. Special thanks to an important friend and associate who stuck by with me all the way, Jie Fu from our endless days of programming (and scribbling equations) during your PhD years to now as a postdoc in MILA \ud83d\ude42And Ilija Ilievski from those years till now too! Big thanks to the people who believed in the future of AI and asset management and made this happen with the original team Damien Loh, Atsuo Ogaki, and Samuel Chen from ensemblecap.ai Thanks to new found close friend Alfredo Canziani who motivated me on open-source education further and supporting each other \ud83d\ude42 And thanks to Moustapha Cisse for setting up an amazing initiative in Africa with #AMMI to kickstart a future of AI by Africa and trusted your students with our teaching. And I'm on my starting journey to help as many students of your as possible by guiding them on learning like Merghaney Mohammed and Enoch Tetteh. Thanks to Soumith Chintala for spearheading #PyTorch and it's now in production running critical functions in finance (like my firm) and tech. Thanks to Marek Bardo\u0144ski where our friendship started when you were in #NVIDIA and we've supported each other ever since in our learning journey. Thanks to #NVIDIA for allowing me to lead deep learning workshops under DLI in NUS all these years. Thanks to my mentors and everyone in #PhilipYeoInitiative like Abel Ang, Seok Lin Khoo, Kiren Kumar, Claire Cheong, Chong Siak Ching, and Kai Hoe Tan Last thanks to everyone in #SingaporeComputerSociety like Jason Chen for being there when I was nervous and Howie Lau for spearheading tech from start till now. And to a senior big award winner I've a lot to learn and also from NUS: Jeffrey Tiong! Thanks to forward-looking civil servants and minister like Minister S Iswaran for gracing the event and delivering a speech on digital readiness and skills empowerment for Singaporeans. Thanks to my family for being by my side Carine Chng, Ng William, and Adabelle Ng for giving me the freedom and support to pursue whatever I wanted even from young. To the future of AI! Cheers, Ritchie","title":"IT Youth Leader of The Year 2019, March 2019"},{"location":"news/it_youth_leader_2019_03_11/#it-youth-leader-of-the-year-2019-award","text":"","title":"IT Youth Leader of the Year 2019 Award"},{"location":"news/it_youth_leader_2019_03_11/#we-got-it","text":"I am very happy to be the IT Youth Leader of the Year 2019 in Singapore for my industry and non-profit contribution in AI at Deep Learning Wizard , ensemblecap.ai, NUS, and NVIDIA! I would like to thank the professor who changed my life, believed in me and nominated me for consideration, Professor Chua Tat-Seng from NExT Centre, NUS School of Computing. Big thanks to everyone who helped for this in NExT Centre particularly Choon Meng, Tek Min, Chua Yi Hao, and Eugene Tan. Special thanks to an important friend and associate who stuck by with me all the way, Jie Fu from our endless days of programming (and scribbling equations) during your PhD years to now as a postdoc in MILA \ud83d\ude42And Ilija Ilievski from those years till now too! Big thanks to the people who believed in the future of AI and asset management and made this happen with the original team Damien Loh, Atsuo Ogaki, and Samuel Chen from ensemblecap.ai Thanks to new found close friend Alfredo Canziani who motivated me on open-source education further and supporting each other \ud83d\ude42 And thanks to Moustapha Cisse for setting up an amazing initiative in Africa with #AMMI to kickstart a future of AI by Africa and trusted your students with our teaching. And I'm on my starting journey to help as many students of your as possible by guiding them on learning like Merghaney Mohammed and Enoch Tetteh. Thanks to Soumith Chintala for spearheading #PyTorch and it's now in production running critical functions in finance (like my firm) and tech. Thanks to Marek Bardo\u0144ski where our friendship started when you were in #NVIDIA and we've supported each other ever since in our learning journey. Thanks to #NVIDIA for allowing me to lead deep learning workshops under DLI in NUS all these years. Thanks to my mentors and everyone in #PhilipYeoInitiative like Abel Ang, Seok Lin Khoo, Kiren Kumar, Claire Cheong, Chong Siak Ching, and Kai Hoe Tan Last thanks to everyone in #SingaporeComputerSociety like Jason Chen for being there when I was nervous and Howie Lau for spearheading tech from start till now. And to a senior big award winner I've a lot to learn and also from NUS: Jeffrey Tiong! Thanks to forward-looking civil servants and minister like Minister S Iswaran for gracing the event and delivering a speech on digital readiness and skills empowerment for Singaporeans. Thanks to my family for being by my side Carine Chng, Ng William, and Adabelle Ng for giving me the freedom and support to pursue whatever I wanted even from young. To the future of AI! Cheers, Ritchie","title":"We got it!"},{"location":"news/nanjing_next_nus_tsinghua_ai_finance_healthcare_2018_11_01/","text":"NExT++ AI in Finance and Healthcare Workshop 2018 \u00b6 Recap \u00b6 This is NExT++\u2019s 3 rd Workshop and this time we are looking at the applications and development of AI related technologies in the healthcare and finance verticals with the theme \u201cAI in Health and Finance\u201d Visual summary of my talk on AI and Unstructured Analytics in Fintech. Doesn't contain everything, but whatever the non-technical designers could come up with in real-time. Really amazing so kudos to them. Really thankful to the whole team in my former lab, NExT++ and a close friend, Choon Meng, for making all the arrangements. Also thanks to Tek Min, Yi Hao and everyone else. Super happy to catch up with Prof Tat-Seng Chua (KITHCT Chair Professor at the School of Computing), Prof Sun Maosong (Dean of Department of Computer Science and Technology, Tsinghua University), and Prof Dame Wendy Hall (Director of the Web Science Institute, University of Southampton). Cheers, Ritchie","title":"NExT++ AI in Healthcare and Finance, Nanjing, November 2018"},{"location":"news/nanjing_next_nus_tsinghua_ai_finance_healthcare_2018_11_01/#next-ai-in-finance-and-healthcare-workshop-2018","text":"","title":"NExT++ AI in Finance and Healthcare Workshop 2018"},{"location":"news/nanjing_next_nus_tsinghua_ai_finance_healthcare_2018_11_01/#recap","text":"This is NExT++\u2019s 3 rd Workshop and this time we are looking at the applications and development of AI related technologies in the healthcare and finance verticals with the theme \u201cAI in Health and Finance\u201d Visual summary of my talk on AI and Unstructured Analytics in Fintech. Doesn't contain everything, but whatever the non-technical designers could come up with in real-time. Really amazing so kudos to them. Really thankful to the whole team in my former lab, NExT++ and a close friend, Choon Meng, for making all the arrangements. Also thanks to Tek Min, Yi Hao and everyone else. Super happy to catch up with Prof Tat-Seng Chua (KITHCT Chair Professor at the School of Computing), Prof Sun Maosong (Dean of Department of Computer Science and Technology, Tsinghua University), and Prof Dame Wendy Hall (Director of the Web Science Institute, University of Southampton). Cheers, Ritchie","title":"Recap"},{"location":"news/news/","text":"Welcome to our Blog \u00b6 Here, we post news related to Deep Learning Wizard's releases, features and achievements Notable News \u00b6 Oral Presentation for AI for Social Good Workshop ICML, Long Beach, Los Angeles, USA, June 2019 IT Youth Leader of The Year, Singapore Computer Society (SCS), Singapore, March 2019 Foundations of Deep Learning, African Masters of Machine Intelligence (AMMI), Google & Facebook, Kigali, Rwanda, November 2018 Invited to Facebook PyTorch Developer Conference, San Francisco, USA, September 2018 Led NUS-MIT-NUHS Datathon NVIDIA Image Recognition Workshop, Singapore, July 2018 Featured on PyTorch Website, January 2018 Led NVIDIA Self-Driving Cars and Healthcare Talk, Singapore, June 2017 Clinched NVIDIA Inception Partner Status, Singapore, May 2017","title":"Welcome"},{"location":"news/news/#welcome-to-our-blog","text":"Here, we post news related to Deep Learning Wizard's releases, features and achievements","title":"Welcome to our Blog"},{"location":"news/news/#notable-news","text":"Oral Presentation for AI for Social Good Workshop ICML, Long Beach, Los Angeles, USA, June 2019 IT Youth Leader of The Year, Singapore Computer Society (SCS), Singapore, March 2019 Foundations of Deep Learning, African Masters of Machine Intelligence (AMMI), Google & Facebook, Kigali, Rwanda, November 2018 Invited to Facebook PyTorch Developer Conference, San Francisco, USA, September 2018 Led NUS-MIT-NUHS Datathon NVIDIA Image Recognition Workshop, Singapore, July 2018 Featured on PyTorch Website, January 2018 Led NVIDIA Self-Driving Cars and Healthcare Talk, Singapore, June 2017 Clinched NVIDIA Inception Partner Status, Singapore, May 2017","title":"Notable News"},{"location":"news/nvidia_nus_mit_datathon_2018_07_05/","text":"NVIDIA Workshop at NUS-MIT-NUHS Datathon \u00b6 Image Recognition Workshop by Ritchie Ng \u00b6 The NVIDIA Deep Learning Institute (DLI) trains developers, data scientists, and researchers on how to use artificial intelligence to solve real-world problems across a wide range of domains. In the deep learning courses, you\u2019ll learn how to train, optimize, and deploy neural networks using the latest tools, frameworks, and techniques for deep learning. In \u201cImage Classification with DIGITS\u201d mini-course, you will learn how to train a deep neural network to recognize handwritten digits by loading image data into a training environment, choosing and training a network, testing with new data, and iterating to improve performance. Link to the NUS-MIT-NUHS Datathon workshop .","title":"NUS-MIT-NUHS NVIDIA Image Recognition Workshop, Singapore, July 2018"},{"location":"news/nvidia_nus_mit_datathon_2018_07_05/#nvidia-workshop-at-nus-mit-nuhs-datathon","text":"","title":"NVIDIA Workshop at NUS-MIT-NUHS Datathon"},{"location":"news/nvidia_nus_mit_datathon_2018_07_05/#image-recognition-workshop-by-ritchie-ng","text":"The NVIDIA Deep Learning Institute (DLI) trains developers, data scientists, and researchers on how to use artificial intelligence to solve real-world problems across a wide range of domains. In the deep learning courses, you\u2019ll learn how to train, optimize, and deploy neural networks using the latest tools, frameworks, and techniques for deep learning. In \u201cImage Classification with DIGITS\u201d mini-course, you will learn how to train a deep neural network to recognize handwritten digits by loading image data into a training environment, choosing and training a network, testing with new data, and iterating to improve performance. Link to the NUS-MIT-NUHS Datathon workshop .","title":"Image Recognition Workshop by Ritchie Ng"},{"location":"news/nvidia_self_driving_cars_talk_2017_06_21/","text":"NVIDIA Self-Driving Cars and Healthcare Workshop \u00b6 Hosted by Ritchie Ng \u00b6 A talk by Marek Bardo\u0144ski, Senior Deep Learning Research Engineer, NVIDIA Switzerland. This is hosted by Ritchie Ng, Deep Learning Researcher, NExT Search Centre, NUS. We will be touching on cutting-edge applications of deep learning for self-driving cars and medical diagnostics. Also there will be a tutorial followed by networking with deep learning researchers from NUS and NVIDIA. Details: Wednesday, June 21 st 1:00 PM to 3:30 PM The Hangar by NUS Enterprise 21 Heng Mui Keng Terrace, Singapore 119613","title":"NVIDIA Self Driving Cars & Healthcare Talk, Singapore, June 2017"},{"location":"news/nvidia_self_driving_cars_talk_2017_06_21/#nvidia-self-driving-cars-and-healthcare-workshop","text":"","title":"NVIDIA Self-Driving Cars and Healthcare Workshop"},{"location":"news/nvidia_self_driving_cars_talk_2017_06_21/#hosted-by-ritchie-ng","text":"A talk by Marek Bardo\u0144ski, Senior Deep Learning Research Engineer, NVIDIA Switzerland. This is hosted by Ritchie Ng, Deep Learning Researcher, NExT Search Centre, NUS. We will be touching on cutting-edge applications of deep learning for self-driving cars and medical diagnostics. Also there will be a tutorial followed by networking with deep learning researchers from NUS and NVIDIA. Details: Wednesday, June 21 st 1:00 PM to 3:30 PM The Hangar by NUS Enterprise 21 Heng Mui Keng Terrace, Singapore 119613","title":"Hosted by Ritchie Ng"},{"location":"ood/intro/","text":"Out-of-distribution Data \u00b6 This is a persistent and critical production issue present in any machine learning and deep learning systems, no matter how good the models were trained. A single out-of-distribution sample data fed into a well-trained model can result in a few problematic outcomes, and in production-level systems they become fat-tailed critical risks that have outsized consequences. Problematic consequences of out-of-distribution data False positive & False Negative : prediction does not match the ground truth. Unreliable True positive & True Negative : prediction matches the ground truth but due to samples being out-of-distribution, the performance of these predictions becomes very erratic.","title":"Out-of-distribution Data"},{"location":"ood/intro/#out-of-distribution-data","text":"This is a persistent and critical production issue present in any machine learning and deep learning systems, no matter how good the models were trained. A single out-of-distribution sample data fed into a well-trained model can result in a few problematic outcomes, and in production-level systems they become fat-tailed critical risks that have outsized consequences. Problematic consequences of out-of-distribution data False positive & False Negative : prediction does not match the ground truth. Unreliable True positive & True Negative : prediction matches the ground truth but due to samples being out-of-distribution, the performance of these predictions becomes very erratic.","title":"Out-of-distribution Data"},{"location":"programming/intro/","text":"Programming \u00b6 We'll be covering C++, Python, Bash and more for end-to-end AI deployments. Packages and Languages you will learn to use C++ Python Bash Matplotlib Javascript Electron Work in progress This open-source portion is still a work in progress. Stay tuned while we gradually upload our tutorials and notes. Feel free to contact Ritchie Ng if you would like to contribute via our Facebook page.","title":"Introduction"},{"location":"programming/intro/#programming","text":"We'll be covering C++, Python, Bash and more for end-to-end AI deployments. Packages and Languages you will learn to use C++ Python Bash Matplotlib Javascript Electron Work in progress This open-source portion is still a work in progress. Stay tuned while we gradually upload our tutorials and notes. Feel free to contact Ritchie Ng if you would like to contribute via our Facebook page.","title":"Programming"},{"location":"programming/bash/bash/","text":"Bash \u00b6 Run Bash Files You can find the bash code files for this section in this link . Creating and Running a Bash File \u00b6 Create bash file \u00b6 touch hello_world.sh Edit bash file with Hello World \u00b6 You can edit with anything like Vim, Sublime, etc. echo Hello World! Run bash file \u00b6 bash hello_world.sh This will print out, in your bash: Hello World! Calculating \u00b6 # Add echo $(( 10 + 10 )) # Subtract echo $(( 10 - 10 )) # Divide echo $(( 10 / 10 )) # Multiple echo $(( 10 * 10 )) # Modulo echo $(( 10 % 10 )) # Multiple Operations: Divide and Add echo $(( 10 / 10 + 10 )) 20 0 1 100 0 11 Loops and Conditional \u00b6 For Loop \u00b6 for i in 'A' 'B' 'C' do echo $i done A B C For Loop With Range \u00b6 This will echo the digits 0 to 10 without explicitly requiring to define the whole range of numbers/alphabets like above. for (( i = 0 ; i< = 10 ; i++ )) ; do echo $i done 0 1 2 3 4 5 6 7 8 9 10 If Else Conditional \u00b6 This is a simple if-else to check if the day of the week is 5, meaning if it is a Friday. day = $( date +%u ) if [ $day == 5 ] ; then echo \"Friday is here!\" else echo \"Friday is not here :(\" echo \"Today is day $day of the week\" fi Sequentially Running of Python Scripts \u00b6 This snippet allows you to to run 3 python scripts sequentially, waiting for each to finish before proceeding to the next. python script_1.py wait python script_2.py wait python script_3.py wait echo \"Finished running all 3 scripts in sequence!\" Parallel Running of Python Scripts \u00b6 python script_1.py && script_2.py && script_3.py wait echo \"Finished running all 3 scripts in parallel in sequence\" Reading and Writing Operations \u00b6 Reading logs and texts \u00b6 Create a text file called random_text.txt with the following contents Row 1 Row 2 Row 3 Row 4 Row 5 Row 6 Row 7 Row 8 Row 9 Row 10 Then run the following command to read it in bash then print it. text_file = $( cat random_text.txt ) echo $text_file Row 1 Row 2 Row 3 Row 4 Row 5 Row 6 Row 7 Row 8 Row 9 Row 10 Date Operations \u00b6 Getting Current Date \u00b6 This will return the date in the format YYYY-MM-DD for example 2019-06-03. DATE = ` date +%Y-%m-%d ` echo $DATE Getting Current Day of Week \u00b6 This will return 1, 2, 3, 4, 5, 6, 7 depending on the day of the week. DAY = $( date +%u ) echo $DAY Changing System Dates By 1 Day \u00b6 You can change system dates based on this. Surprisingly, you'll find it useful for testing an environment for deployments in the next day and then shifting it back to the actual day. sudo date -s 'next day' sudo date -s 'yesterday' If you are running some tests via bash and want to disable typing in password you can edit the sudoer file via sudo visudo and adding the following line. Only use sudo visudo and nothing else, as they've a special syntax. <username> ALL =( ALL ) NOPASSWD: /bin/date To find out your username, simply just run the command whoami . Jupyter Utility Commands \u00b6 Convert Notebook to HTML/Markdown \u00b6 jupyter nbconvert --to markdown python.ipynb jupyter nbconvert --to html python.ipynb Bash Convenient Commands \u00b6 List directories only \u00b6 ls -d */ List non-directories only \u00b6 ls -p | grep -v '/$' Check IP \u00b6 ifconfig | sed -En \"s/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\\.){3}[0-9]*).*/\\2/p\" Check internet speed \u00b6 ethtool eno1 Check disk space \u00b6 df -h Check ubuntu version \u00b6 lsb_release -a Check truncated system logs \u00b6 tail /var/log/syslog Check CUDA version \u00b6 nvcc -V Check cuDNN version \u00b6 cat /usr/local/cuda/include/cudnn.h | grep CUDNN_MAJOR -A 2 Check username \u00b6 whoami Untar file \u00b6 tar -xvzf file_name Open PDF file \u00b6 gvfs-open file_name Download file from link rapidly with aria \u00b6 aria2c -x16 -c url_link Kill all python processes \u00b6 ps ax | grep python | cut -c1-5 | xargs kill -9 Install .deb files \u00b6 sudo apt-get install -f file_name.deb Empty thrash \u00b6 sudo apt-get install trash-cli thrash-empty Fix Git Permissions \u00b6 There are times whether you're on Windows (frequent), Mac, or Linux, you might encounter a permission error although you have obviously set it up correctly and it was last working. This is quick fix you can run. eval ` ssh-agent -s ` && ssh-add ~/.ssh/github If you want, you can even make an alias fixgit and quickly call it in bash to make it easier to fix whenever you face this issue. It's a quick fix, there're more permnanent ways to fix it if you want. But this is a quick fix section. In your .bashrc or .zshrc , include the following: alias fixgit = 'eval `ssh-agent -s` && ssh-add ~/.ssh/github' Get Git Repo Information \u00b6 curl https://api.github.com/<user>/<repo_name> Conda Commands \u00b6 Check conda environment \u00b6 conda env list Create conda kernel \u00b6 conda create -n kernel_name python = 3 .6 source activate kernel_name Install conda kernel \u00b6 conda install ipykernel source activate kernel_name python -m ipykernel install --user --name kernel_name --display-name kernel_name Remove conda kernel \u00b6 conda env remove -n kernel_name Recovering problematic conda installation \u00b6 # Download miniconda according to your environment # Link: https://docs.conda.io/en/latest/miniconda.html # Backup existing miniconda environment that may have problems mv miniconda3 miniconda3_backup # Install miniconda bash Miniconda3-latest-Linux-x86_64.sh # Restore old environment settings rsync -a miniconda3_backup/ miniconda3/ Internet Operations \u00b6 Checking Internet Availability \u00b6 This script will return whether your internet is fine or not without using pings. Pings can often be rendered unusable when the network administrator disables ICMP to prevent the origination of ping floods from the data centre. if nc -zw1 google.com 443 ; then echo \"INTERNET: OK\" else echo \"INTERNET: NOT OK\" fi Cron Operations \u00b6 Edit Cron \u00b6 Formatting follows this syntax with full credits on this beautiful diagram to fedorqui from Stack Overflow : \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 minute (0 - 59) \u2502 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 hour (0 - 23) \u2502 \u2502 \u250c\u2500\u2500\u2500\u2500\u2500\u2500 day of month (1 - 31) \u2502 \u2502 \u2502 \u250c\u2500\u2500\u2500\u2500 month (1 - 12) \u2502 \u2502 \u2502 \u2502 \u250c\u2500\u2500 day of week (0 - 6 => Sunday - Saturday, or \u2502 \u2502 \u2502 \u2502 \u2502 1 - 7 => Monday - Sunday) \u2193 \u2193 \u2193 \u2193 \u2193 * * * * * command to be executed Edit cron with this command: sudo crontab -e List Cron \u00b6 sudo crontab -l Status, Start, Stop and Restart \u00b6 sudo service cron status sudo service cron stop sudo service cron start sudo service cron restart Cron Debugging \u00b6 Install postfix for local routing of errors (choose local option): sudo apt-get install postfix Restart cron to see for any errors posted (if not errors, there will be no file, be patient before errors are posted): sudo cat /var/mail/root Cron Bash Fix \u00b6 Cron uses /bin/sh as the default shell. Typically you would realize you're using /bin/bash as your shell, so this typically needs to be rectified before you can use cron to schedule commands as if it were your usual bash. Edit your cron file via sudo crontab -e and paste the following lines at the end of the file prior to your command like so. Take note for PATH , you've to paste the output of echo PATH=$PATH in there instead! SHELL=/bin/bash PATH=/usr/lib.... # Your command schedule here! Cron Conda Environment \u00b6 This is an example of enabling an anaconda environment, for example the default base , and running a python script. Take note you need to put your python script in the right directory or simply navigate to that path with cd prior to \"$(conda shell.bash hook)\" . SHELL=/bin/bash PATH=/usr/lib.... * * * * 1-5 eval \"$(conda shell.bash hook)\" && conda activate base && python python_script_name.py Cron Running Processes \u00b6 Some times you want to see the status of running tasks and may want to get the PID to end it. This is a very handy command. ps - o pid , sess , cmd afx | egrep - A20 \"( |/)cron( -f)?$\" You can get the PID of the cron process and then end it with sudo pkill -s <PID> Hardware Information \u00b6 Comprehensive CPU Information \u00b6 cat /proc/cpuinfo Number of CPU Threads \u00b6 !grep -c ^processor /proc/cpuinfo or nproc CPU Model Name \u00b6 !cat /proc/cpuinfo | grep \"model name\" Check Available RAM \u00b6 In MB \u00b6 free -m In GB \u00b6 free -g","title":"Bash"},{"location":"programming/bash/bash/#bash","text":"Run Bash Files You can find the bash code files for this section in this link .","title":"Bash"},{"location":"programming/bash/bash/#creating-and-running-a-bash-file","text":"","title":"Creating and Running a Bash File"},{"location":"programming/bash/bash/#create-bash-file","text":"touch hello_world.sh","title":"Create bash file"},{"location":"programming/bash/bash/#edit-bash-file-with-hello-world","text":"You can edit with anything like Vim, Sublime, etc. echo Hello World!","title":"Edit bash file with Hello World"},{"location":"programming/bash/bash/#run-bash-file","text":"bash hello_world.sh This will print out, in your bash: Hello World!","title":"Run bash file"},{"location":"programming/bash/bash/#calculating","text":"# Add echo $(( 10 + 10 )) # Subtract echo $(( 10 - 10 )) # Divide echo $(( 10 / 10 )) # Multiple echo $(( 10 * 10 )) # Modulo echo $(( 10 % 10 )) # Multiple Operations: Divide and Add echo $(( 10 / 10 + 10 )) 20 0 1 100 0 11","title":"Calculating"},{"location":"programming/bash/bash/#loops-and-conditional","text":"","title":"Loops and Conditional"},{"location":"programming/bash/bash/#for-loop","text":"for i in 'A' 'B' 'C' do echo $i done A B C","title":"For Loop"},{"location":"programming/bash/bash/#for-loop-with-range","text":"This will echo the digits 0 to 10 without explicitly requiring to define the whole range of numbers/alphabets like above. for (( i = 0 ; i< = 10 ; i++ )) ; do echo $i done 0 1 2 3 4 5 6 7 8 9 10","title":"For Loop With Range"},{"location":"programming/bash/bash/#if-else-conditional","text":"This is a simple if-else to check if the day of the week is 5, meaning if it is a Friday. day = $( date +%u ) if [ $day == 5 ] ; then echo \"Friday is here!\" else echo \"Friday is not here :(\" echo \"Today is day $day of the week\" fi","title":"If Else Conditional"},{"location":"programming/bash/bash/#sequentially-running-of-python-scripts","text":"This snippet allows you to to run 3 python scripts sequentially, waiting for each to finish before proceeding to the next. python script_1.py wait python script_2.py wait python script_3.py wait echo \"Finished running all 3 scripts in sequence!\"","title":"Sequentially Running of Python Scripts"},{"location":"programming/bash/bash/#parallel-running-of-python-scripts","text":"python script_1.py && script_2.py && script_3.py wait echo \"Finished running all 3 scripts in parallel in sequence\"","title":"Parallel Running of Python Scripts"},{"location":"programming/bash/bash/#reading-and-writing-operations","text":"","title":"Reading and Writing Operations"},{"location":"programming/bash/bash/#reading-logs-and-texts","text":"Create a text file called random_text.txt with the following contents Row 1 Row 2 Row 3 Row 4 Row 5 Row 6 Row 7 Row 8 Row 9 Row 10 Then run the following command to read it in bash then print it. text_file = $( cat random_text.txt ) echo $text_file Row 1 Row 2 Row 3 Row 4 Row 5 Row 6 Row 7 Row 8 Row 9 Row 10","title":"Reading logs and texts"},{"location":"programming/bash/bash/#date-operations","text":"","title":"Date Operations"},{"location":"programming/bash/bash/#getting-current-date","text":"This will return the date in the format YYYY-MM-DD for example 2019-06-03. DATE = ` date +%Y-%m-%d ` echo $DATE","title":"Getting Current Date"},{"location":"programming/bash/bash/#getting-current-day-of-week","text":"This will return 1, 2, 3, 4, 5, 6, 7 depending on the day of the week. DAY = $( date +%u ) echo $DAY","title":"Getting Current Day of Week"},{"location":"programming/bash/bash/#changing-system-dates-by-1-day","text":"You can change system dates based on this. Surprisingly, you'll find it useful for testing an environment for deployments in the next day and then shifting it back to the actual day. sudo date -s 'next day' sudo date -s 'yesterday' If you are running some tests via bash and want to disable typing in password you can edit the sudoer file via sudo visudo and adding the following line. Only use sudo visudo and nothing else, as they've a special syntax. <username> ALL =( ALL ) NOPASSWD: /bin/date To find out your username, simply just run the command whoami .","title":"Changing System Dates By 1 Day"},{"location":"programming/bash/bash/#jupyter-utility-commands","text":"","title":"Jupyter Utility Commands"},{"location":"programming/bash/bash/#convert-notebook-to-htmlmarkdown","text":"jupyter nbconvert --to markdown python.ipynb jupyter nbconvert --to html python.ipynb","title":"Convert Notebook to HTML/Markdown"},{"location":"programming/bash/bash/#bash-convenient-commands","text":"","title":"Bash Convenient Commands"},{"location":"programming/bash/bash/#list-directories-only","text":"ls -d */","title":"List directories only"},{"location":"programming/bash/bash/#list-non-directories-only","text":"ls -p | grep -v '/$'","title":"List non-directories only"},{"location":"programming/bash/bash/#check-ip","text":"ifconfig | sed -En \"s/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\\.){3}[0-9]*).*/\\2/p\"","title":"Check IP"},{"location":"programming/bash/bash/#check-internet-speed","text":"ethtool eno1","title":"Check internet speed"},{"location":"programming/bash/bash/#check-disk-space","text":"df -h","title":"Check disk space"},{"location":"programming/bash/bash/#check-ubuntu-version","text":"lsb_release -a","title":"Check ubuntu version"},{"location":"programming/bash/bash/#check-truncated-system-logs","text":"tail /var/log/syslog","title":"Check truncated system logs"},{"location":"programming/bash/bash/#check-cuda-version","text":"nvcc -V","title":"Check CUDA version"},{"location":"programming/bash/bash/#check-cudnn-version","text":"cat /usr/local/cuda/include/cudnn.h | grep CUDNN_MAJOR -A 2","title":"Check cuDNN version"},{"location":"programming/bash/bash/#check-username","text":"whoami","title":"Check username"},{"location":"programming/bash/bash/#untar-file","text":"tar -xvzf file_name","title":"Untar file"},{"location":"programming/bash/bash/#open-pdf-file","text":"gvfs-open file_name","title":"Open PDF file"},{"location":"programming/bash/bash/#download-file-from-link-rapidly-with-aria","text":"aria2c -x16 -c url_link","title":"Download file from link rapidly with aria"},{"location":"programming/bash/bash/#kill-all-python-processes","text":"ps ax | grep python | cut -c1-5 | xargs kill -9","title":"Kill all python processes"},{"location":"programming/bash/bash/#install-deb-files","text":"sudo apt-get install -f file_name.deb","title":"Install .deb files"},{"location":"programming/bash/bash/#empty-thrash","text":"sudo apt-get install trash-cli thrash-empty","title":"Empty thrash"},{"location":"programming/bash/bash/#fix-git-permissions","text":"There are times whether you're on Windows (frequent), Mac, or Linux, you might encounter a permission error although you have obviously set it up correctly and it was last working. This is quick fix you can run. eval ` ssh-agent -s ` && ssh-add ~/.ssh/github If you want, you can even make an alias fixgit and quickly call it in bash to make it easier to fix whenever you face this issue. It's a quick fix, there're more permnanent ways to fix it if you want. But this is a quick fix section. In your .bashrc or .zshrc , include the following: alias fixgit = 'eval `ssh-agent -s` && ssh-add ~/.ssh/github'","title":"Fix Git Permissions"},{"location":"programming/bash/bash/#get-git-repo-information","text":"curl https://api.github.com/<user>/<repo_name>","title":"Get Git Repo Information"},{"location":"programming/bash/bash/#conda-commands","text":"","title":"Conda Commands"},{"location":"programming/bash/bash/#check-conda-environment","text":"conda env list","title":"Check conda environment"},{"location":"programming/bash/bash/#create-conda-kernel","text":"conda create -n kernel_name python = 3 .6 source activate kernel_name","title":"Create conda kernel"},{"location":"programming/bash/bash/#install-conda-kernel","text":"conda install ipykernel source activate kernel_name python -m ipykernel install --user --name kernel_name --display-name kernel_name","title":"Install conda kernel"},{"location":"programming/bash/bash/#remove-conda-kernel","text":"conda env remove -n kernel_name","title":"Remove conda kernel"},{"location":"programming/bash/bash/#recovering-problematic-conda-installation","text":"# Download miniconda according to your environment # Link: https://docs.conda.io/en/latest/miniconda.html # Backup existing miniconda environment that may have problems mv miniconda3 miniconda3_backup # Install miniconda bash Miniconda3-latest-Linux-x86_64.sh # Restore old environment settings rsync -a miniconda3_backup/ miniconda3/","title":"Recovering problematic conda installation"},{"location":"programming/bash/bash/#internet-operations","text":"","title":"Internet Operations"},{"location":"programming/bash/bash/#checking-internet-availability","text":"This script will return whether your internet is fine or not without using pings. Pings can often be rendered unusable when the network administrator disables ICMP to prevent the origination of ping floods from the data centre. if nc -zw1 google.com 443 ; then echo \"INTERNET: OK\" else echo \"INTERNET: NOT OK\" fi","title":"Checking Internet Availability"},{"location":"programming/bash/bash/#cron-operations","text":"","title":"Cron Operations"},{"location":"programming/bash/bash/#edit-cron","text":"Formatting follows this syntax with full credits on this beautiful diagram to fedorqui from Stack Overflow : \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 minute (0 - 59) \u2502 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 hour (0 - 23) \u2502 \u2502 \u250c\u2500\u2500\u2500\u2500\u2500\u2500 day of month (1 - 31) \u2502 \u2502 \u2502 \u250c\u2500\u2500\u2500\u2500 month (1 - 12) \u2502 \u2502 \u2502 \u2502 \u250c\u2500\u2500 day of week (0 - 6 => Sunday - Saturday, or \u2502 \u2502 \u2502 \u2502 \u2502 1 - 7 => Monday - Sunday) \u2193 \u2193 \u2193 \u2193 \u2193 * * * * * command to be executed Edit cron with this command: sudo crontab -e","title":"Edit Cron"},{"location":"programming/bash/bash/#list-cron","text":"sudo crontab -l","title":"List Cron"},{"location":"programming/bash/bash/#status-start-stop-and-restart","text":"sudo service cron status sudo service cron stop sudo service cron start sudo service cron restart","title":"Status, Start, Stop and Restart"},{"location":"programming/bash/bash/#cron-debugging","text":"Install postfix for local routing of errors (choose local option): sudo apt-get install postfix Restart cron to see for any errors posted (if not errors, there will be no file, be patient before errors are posted): sudo cat /var/mail/root","title":"Cron Debugging"},{"location":"programming/bash/bash/#cron-bash-fix","text":"Cron uses /bin/sh as the default shell. Typically you would realize you're using /bin/bash as your shell, so this typically needs to be rectified before you can use cron to schedule commands as if it were your usual bash. Edit your cron file via sudo crontab -e and paste the following lines at the end of the file prior to your command like so. Take note for PATH , you've to paste the output of echo PATH=$PATH in there instead! SHELL=/bin/bash PATH=/usr/lib.... # Your command schedule here!","title":"Cron Bash Fix"},{"location":"programming/bash/bash/#cron-conda-environment","text":"This is an example of enabling an anaconda environment, for example the default base , and running a python script. Take note you need to put your python script in the right directory or simply navigate to that path with cd prior to \"$(conda shell.bash hook)\" . SHELL=/bin/bash PATH=/usr/lib.... * * * * 1-5 eval \"$(conda shell.bash hook)\" && conda activate base && python python_script_name.py","title":"Cron Conda Environment"},{"location":"programming/bash/bash/#cron-running-processes","text":"Some times you want to see the status of running tasks and may want to get the PID to end it. This is a very handy command. ps - o pid , sess , cmd afx | egrep - A20 \"( |/)cron( -f)?$\" You can get the PID of the cron process and then end it with sudo pkill -s <PID>","title":"Cron Running Processes"},{"location":"programming/bash/bash/#hardware-information","text":"","title":"Hardware Information"},{"location":"programming/bash/bash/#comprehensive-cpu-information","text":"cat /proc/cpuinfo","title":"Comprehensive CPU Information"},{"location":"programming/bash/bash/#number-of-cpu-threads","text":"!grep -c ^processor /proc/cpuinfo or nproc","title":"Number of CPU Threads"},{"location":"programming/bash/bash/#cpu-model-name","text":"!cat /proc/cpuinfo | grep \"model name\"","title":"CPU Model Name"},{"location":"programming/bash/bash/#check-available-ram","text":"","title":"Check Available RAM"},{"location":"programming/bash/bash/#in-mb","text":"free -m","title":"In MB"},{"location":"programming/bash/bash/#in-gb","text":"free -g","title":"In GB"},{"location":"programming/cpp/cpp/","text":"C++ \u00b6 Run Jupyter Notebook You can run the code for this section in this jupyter notebook link . Installation of Interactive C++17 \u00b6 Xeus-Cling is a game-changer where similar to Python Jupyter Notebooks, we can run C++ Jupyter Notebooks now. Run the following bash commands in sequence to create a C++ kernel for Jupyter Notebook. conda create -n cpp source activate cpp conda install -c conda-forge xeus-cling jupyter kernelspec install --user /home/ritchie/miniconda3/envs/cpp/share/jupyter/kernels/xcpp17 jupyter notebook Printing \u00b6 Printing Single Line \u00b6 // When you compile, the preprocessor runs and acts on all line with the pound key first // This is a preprocessor instruction that // essentially places the file with the name iostream into this spot #include <iostream> // Print one line std :: cout << \"Using Xeus Cling\" << std :: endl ; Using Xeus Cling Printing 2 Lines \u00b6 // Print two lines std :: cout << \"First Line \\n Second Line\" << std :: endl ; First Line Second Line Printing with Tabs \u00b6 // Print numbers with nicely formatted tabs std :: cout << \"One hundred: \\t \" ; std :: cout << ( float ) 1000 / 10 << std :: endl ; std :: cout << \"Two hundred: \\t \" ; std :: cout << ( double ) 2000 / 10 << std :: endl ; std :: cout << \"Three hundred: \\t \" ; std :: cout << ( double ) 3000 / 10 << std :: endl ; One hundred: 100 Two hundred: 200 Three hundred: 300 Easier Printing (subjective) with Namespaces \u00b6 Gets irritating to use std in front of cout and endl to keep printing so we can use namespaces using namespace std ; cout << \"No need for messy std::\" << endl ; No need for messy std:: Variables \u00b6 General memory management \u00b6 In C++, you always need to determine each variable's type so the compiler will know how much memory (RAM) to allocate to the variable Types of Variables \u00b6 Type Bytes (Size) Range of Values char 1 0 to 255 or -127 to 127 unsigned char 1 -127 to 127 signed char 1 0 to 255 bool 1 True or False int 4 -2,147,483,648 to 2,147,483,647 unsigned int 4 0 to 4,294,967,295 signed int 4 -2,147,483,648 to 2,147,483,647 short int 2 -32,768 to 32,767 long int 4 -2,147,483,648 to 2,147,483,647 float 4 3.4e-38 to 3.4e38 double 8 1.7e-308 to 1.7e-08 long double 8 1.7e-308 to 1.7e-08 Integer sizes \u00b6 Typical sizes Short integer: 2 bytes (will be smaller than long) Limitation is that the value of this short integer has a max value, you should be cautious of using short integers Long integer: 4 bytes Integer: 2 or 4 bytes Notes Technically these sizes can vary depending on your processor (32/64 bit) and compiler You should not assume the sizes but the hierarchy of sizes will not change (short memory < long memory) Integer size \u00b6 cout << \"Size of an integer: \" << sizeof ( int ); Size of an integer: 4 Short integer size \u00b6 cout << \"Size of a short integer: \" << sizeof ( short int ); Size of a short integer: 2 Long integer size \u00b6 cout << \"Size of an long integer: \" << sizeof ( long int ); Size of an long integer: 8 Unsigned or signed integers \u00b6 Unsigned integers: only can hold positive integers Signed integers: can hold positive/negative integers Signed short integer \u00b6 cout << \"Size of an signed short integer: \" << sizeof ( signed short int ); Size of an signed short integer: 2 Unsigned short integer \u00b6 cout << \"Size of an unsigned short integer: \" << sizeof ( unsigned short int ); Size of an unsigned short integer: 2 Signed long integer \u00b6 cout << \"Size of an signed long integer: \" << sizeof ( signed long int ); Size of an signed long integer: 8 Unsigned long integer \u00b6 cout << \"Size of an unsigned long integer: \" << sizeof ( unsigned long int ); Size of an unsigned long integer: 8 Constants \u00b6 Literal Constants \u00b6 int varOne = 20 ; std :: cout << varOne << std :: endl ; 20 Enumerated Constants \u00b6 This enables you to create a new type! In this example we create a new type directions containing Up , Down , Left , and Right . enum directions { Up , Down , Left , Right }; directions goWhere ; goWhere = Right ; if ( goWhere == Right ) std :: cout << \"Go right\" << std :: endl ; Go right Functions \u00b6 Function Without Return Value \u00b6 Syntax generally follows void FunctionName(argOne, argTwo) to define the function followed by FuntionName() to call the function. // Usage of void when the function does not return anything // In this example, this function prints out the multiplication result of two given numbers void MultiplyTwoNumbers ( int firstNum , int secondNum ) { // Define variable as integer type long int value ; value = firstNum * secondNum ; std :: cout << value << std :: endl ; } Multiply Two Numbers 3 and 2 \u00b6 MultiplyTwoNumbers ( 3 , 2 ) 6 Multiply Two Numbers 6 and 2 \u00b6 MultiplyTwoNumbers ( 6 , 2 ) 12 Aliases Function \u00b6 Say we want the variable value to be of type unsigned short int such that it's 2 bytes and can hold 2x the range of values compared to just short int . We can use typedef as an alias. Type Bytes (Size) Range of Values short int 2 -32,768 to 32,767 unsigned short int 2 0 to 65,536 // Usage of void when the function does not return anything // In this exmaple, this function prints out the multiplication result of two given numbers void MultiplyTwoNumbersWithAlias ( int firstNum , int secondNum ) { // Using an alias typedef unsigned short int ushortint ; // initializing value variable with ushortint type ushortint value ; value = firstNum * secondNum ; std :: cout << value << std :: endl ; } Multiply Two Numbers 10 and 10 \u00b6 MultiplyTwoNumbersWithAlias ( 10 , 10 ) 100 Multiply Two Numbers 1000 and 65 \u00b6 MultiplyTwoNumbersWithAlias ( 1000 , 65 ) 65000 Multiply Two Numbers 1000 and 67 \u00b6 Notice how you don't get 67,000? This is because our variable value of ushortint type can only hold values up to the integer 65,536. What this returns is the remainder of 67,000 - 65,536 = 1464 MultiplyTwoNumbersWithAlias ( 1000 , 67 ) 1464 std :: cout << 67 * 1000 - 65536 << std :: endl ; 1464 Function with Return Value \u00b6 Unlike functions without return values where we use void to declare the function, here we use int to declare our function that returns values. // In this exmaple, this function returns the value of the multiplication of two numbers int MultiplyTwoNumbersNoPrint ( int firstNum , int secondNum ) { // Define variable as integer type long int value ; value = firstNum * secondNum ; return value ; } Call Function \u00b6 // Declare variable with type long int returnValue ; // Call function returnValue = MultiplyTwoNumbersNoPrint ( 10 , 2 ); // Print variable std :: cout << returnValue << std :: endl ; 20 Function Inner Workings \u00b6 Essentially our lines of codes translates to instruction pointers with unique memory addresses. Execution of instruction pointers operates on a \"LIFO\" basis, last in first out. Oversimplifying here, in our example, the last line is taken off first and it follows up // Code Space int varOneTest = 10 ; // Instruction pointer 100 std :: cout << varOneTest << std :: endl ; // Instruction Pointer 102 10 @0x7fa6b7de5460 Arrays \u00b6 An array contains a sequence of elements with the same data type. Creating an Array \u00b6 // This is how you declare an array of 50 elements each of type double double DoubleArray [ 50 ]; Accessing Array's Elements \u00b6 First Element \u00b6 // This access the first array std :: cout << DoubleArray [ 0 ] << std :: endl ; 0 Last Element \u00b6 std :: cout << DoubleArray [ 49 ] << std :: endl ; 0 First 10 Elements \u00b6 // In steps of 1 for ( int i = 0 ; i < 10 ; i ++ ) { // This is how you print a mix of characters and declared variables std :: cout << \"Element \" << i << \" contains \" << DoubleArray [ i ] << std :: endl ; } Element 0 contains 0 Element 1 contains 0 Element 2 contains 0 Element 3 contains 0 Element 4 contains 0 Element 5 contains 0 Element 6 contains 0 Element 7 contains 0 Element 8 contains 0 Element 9 contains 0 // In steps of 2 for ( int i = 0 ; i < 10 ; i += 2 ) { // This is how you print a mix of characters and declared variables std :: cout << \"Element \" << i << \" contains \" << DoubleArray [ i ] << std :: endl ; } Element 0 contains 0 Element 2 contains 0 Element 4 contains 0 Element 6 contains 0 Element 8 contains 0 Going Beyond The Array's Length \u00b6 This will return a warning that it's past the end of the array std :: cout << DoubleArray [ 50 ] << std :: endl ; input_line_36:2:15: warning: array index 50 is past the end of the array ( which contains 50 elements ) [ -Warray-bounds ] std::cout << DoubleArray[50] << std::endl; ^ ~~ input_line_32:3:1: note: array 'DoubleArray ' declared here double DoubleArray [ 50 ] ; ^ 4 .94066e-323 Arrays with Enumeration \u00b6 enum directionsNew { up , down , left , right , individualDirections }; int directionsArray [ individualDirections ] = { 1 , 2 , 3 , 4 }; std :: cout << \"Up value: \\t \" << directionsArray [ up ]; std :: cout << \" \\n Down value: \\t \" << directionsArray [ down ]; std :: cout << \" \\n Left value: \\t \" << directionsArray [ left ]; std :: cout << \" \\n Right value: \\t \" << directionsArray [ right ]; // This is the number of elements in the array std :: cout << \" \\n Num value: \\t \" << sizeof ( directionsArray ) / sizeof ( directionsArray [ 0 ]) << std :: endl ; Up value: 1 Down value: 2 Left value: 3 Right value: 4 Num value: 4 Arrays with >1 Dimension (Tensors) \u00b6 Multi Dimension Array with Numbers \u00b6 // This is how you declare a multi-dimensional array of 5x5 elements each of type double double multiDimArray [ 5 ][ 5 ] = { { 1 , 2 , 3 , 4 , 5 }, { 2 , 2 , 3 , 4 , 5 }, { 3 , 2 , 3 , 4 , 5 }, { 4 , 2 , 3 , 4 , 5 }, { 5 , 2 , 3 , 4 , 5 } }; // Print each row of our 5x5 multi-dimensional array for ( int i = 0 ; i < 5 ; i ++ ) { for ( int j = 0 ; j < 5 ; j ++ ) { std :: cout << multiDimArray [ i ][ j ]; }; std :: cout << \" \\n \" << std :: endl ; }; 12345 22345 32345 42345 52345 Multi Dimension Array with Characters \u00b6 // This is how you declare a multi-dimensional array of 5x5 elements each of type char char multiDimArrayChars [ 5 ][ 5 ] = { { 'a' , 'b' , 'c' , 'd' , 'e' }, { 'b' , 'b' , 'c' , 'd' , 'e' }, { 'c' , 'b' , 'c' , 'd' , 'e' }, { 'd' , 'b' , 'c' , 'd' , 'e' }, { 'e' , 'b' , 'c' , 'd' , 'e' }, }; // Print each row of our 5x5 multi-dimensional array for ( int i = 0 ; i < 5 ; i ++ ) { for ( int j = 0 ; j < 5 ; j ++ ) { std :: cout << multiDimArrayChars [ i ][ j ]; }; std :: cout << \" \\n \" << std :: endl ; }; abcde bbcde cbcde dbcde ebcde Copy Arrays \u00b6 Copy Number Arrays \u00b6 double ArrayNumOne [] = { 1 , 2 , 3 , 4 , 5 }; double ArrayNumTwo [ 5 ]; // Use namespace std so it's cleaner using namespace std ; // Copy array with copy() copy ( begin ( ArrayNumOne ), end ( ArrayNumOne ), begin ( ArrayNumTwo )); // Print double type array with copy() copy ( begin ( ArrayNumTwo ), end ( ArrayNumTwo ), ostream_iterator < double > ( cout , \" \\n \" )); 1 2 3 4 5 Copy String Arrays \u00b6 char ArrayCharOne [] = { 'a' , 'b' , 'c' , 'd' , 'e' }; char ArrayCharTwo [ 5 ]; // Use namespace std so it's cleaner using namespace std ; // Copy array with copy() copy ( begin ( ArrayCharOne ), end ( ArrayCharOne ), begin ( ArrayCharTwo )); // Print char type array with copy() copy ( begin ( ArrayCharTwo ), end ( ArrayCharTwo ), ostream_iterator < char > ( cout , \" \\n \" )); a b c d e Mathematical Operators \u00b6 Add, subtract, multiply, divide and modulus Add \u00b6 double addStatement = 5 + 5 ; std :: cout << addStatement ; 10 Subtract \u00b6 double subtractStatement = 5 - 5 ; std :: cout << subtractStatement ; 0 Divide \u00b6 double divideStatement = 5 / 5 ; std :: cout << divideStatement ; 1 Multiply \u00b6 double multiplyStatement = 5 * 5 ; std :: cout << multiplyStatement ; 25 Modulus \u00b6 Gets the remainder of the division double modulusStatement = 8 % 5 ; std :: cout << modulusStatement ; 3 Exponent \u00b6 Base to the power of something, this requires a new package called <cmath> that we want to include. #include <cmath> void SquareNumber ( int baseNum , int exponentNum ) { // Square the locally scoped variable with 2 int squaredNumber ; squaredNumber = pow ( baseNum , exponentNum ); std :: cout << \"Base of 2 with exponent of 2 gives: \" << squaredNumber << std :: endl ; } SquareNumber ( 2 , 2 ) Base of 2 with exponent of 2 gives: 4 Incrementing/Decrementing \u00b6 3 ways to do this, from least to most verbose Methods \u00b6 Method 1 \u00b6 double idx = 1 ; idx ++ ; std :: cout << idx ; 2 Method 2 \u00b6 idx += 1 ; std :: cout << idx ; 3 Method 3 \u00b6 idx = idx + 1 ; std :: cout << idx ; 4 Prefix/Postfix \u00b6 Prefix \u00b6 This will change both incremented variable and the new variable you assign the incremented variable to Summary: both variables will have the same values // Instantiate double a = 1 ; double b = 1 ; // Print original values cout << \"Old a: \\t \" << a << \" \\n \" ; cout << \"Old b: \\t \" << b << \" \\n \" ; // Prefix increment a = ++ b ; // Print new values cout << \"New a: \\t \" << a << \" \\n \" ; cout << \"New b: \\t \" << b << \" \\n \" ; Old a: 1 Old b: 1 New a: 2 New b: 2 Postfix \u00b6 This will change only the incremented variable but not the variable it's assigned to Summary: incremented variable will change but not the variable it was assigned to // Instantiate double c = 2 ; double d = 2 ; // Print original values cout << \"Old c: \\t \" << c << \" \\n \" ; cout << \"Old d: \\t \" << d << \" \\n \" ; // Prefix increment c = d -- ; // Print new values, notice how only d decremented? c which is what d is assigned to doesn't change. cout << \"New c: \\t \" << c << \" \\n \" ; cout << \"New d: \\t \" << d << \" \\n \" ; Old c: 2 Old d: 2 New c: 2 New d: 1 Conditional Statements \u00b6 If \u00b6 int maxValue = 10 ; // Increment till 10 for ( int i = 0 ; i <= 10 ; i += 2 ) { // Stop if the number reaches 10 (inclusive) if ( i == maxValue ) { cout << \"Reached max value!\" ; cout << \" \\n Value is \" << i << endl ; }; }; Reached max value! Value is 10 Else \u00b6 int newMaxValue = 20 ; // Increment till 10 for ( int i = 0 ; i <= 10 ; i += 2 ) { // Stop if the number reaches 10 (inclusive) if ( i == newMaxValue ) { cout << \"Reached max value!\" ; cout << \" \\n Value is \" << i << endl ; } // Else print current value else { cout << \" \\n Current Value is \" << i << endl ; } } Current Value is 0 Current Value is 2 Current Value is 4 Current Value is 6 Current Value is 8 Current Value is 10 Logical Operators \u00b6 And \u00b6 int varOneNew = 10 ; int varTwo = 10 ; int varCheckOne = 10 ; int varCheckTwo = 5 ; // This should print out if (( varOneNew == varCheckOne ) && ( varTwo == varCheckOne )) { std :: cout << \"Both values equal to 10!\" << std :: endl ; } // This should not print out as varTwo does not equal to 5 if (( varOneNew == varCheckOne ) && ( varTwo == varCheckTwo )) { std :: cout << \"VarOneNew equals to 10, VarTwo equals to 5\" << std :: endl ; } Both values equal to 10! Or \u00b6 // On the contrary, this exact same statement would print out // as VarOne is equal to 10 and we are using an OR operator if (( varOneNew == varCheckOne ) || ( varTwo == varCheckTwo )) { std :: cout << \"VarOneNew equals to 10 or VarTwo equals to 5\" << std :: endl ; } VarOneNew equals to 10 or VarTwo equals to 5 Not \u00b6 // This would print out as VarTwo is not equal to 5 if ( varTwo != varCheckTwo ) { std :: cout << \"VarTwo (10) is not equal to VarCheckTwo (5).\" << std :: endl ; } VarTwo (10) is not equal to VarCheckTwo (5). Getting User Input \u00b6 using namespace std ; long double inputOne , inputTwo ; cout << \"This program multiplies 2 given numbers \\n \" ; cout << \"Enter first number: \\n \" ; cin >> inputOne ; cout << \"Enter second number: \\n \" ; cin >> inputTwo ; cout << \"Multiplication value: \" << inputOne * inputTwo << endl ; This program multiplies 2 given numbers Enter first number: 10 Enter second number: 10 Multiplication value: 100 Loops \u00b6 For Loop \u00b6 for ( int i = 0 ; i < 10 ; i += 1 ) { cout << \"Value of i is: \" << i << endl ; } Value of i is: 0 Value of i is: 1 Value of i is: 2 Value of i is: 3 Value of i is: 4 Value of i is: 5 Value of i is: 6 Value of i is: 7 Value of i is: 8 Value of i is: 9 While Loop \u00b6 int idxWhile = 0 ; while ( idxWhile < 10 ) { idxWhile += 1 ; cout << \"Value of while loop i is: \" << idxWhile << endl ; } Value of while loop i is: 1 Value of while loop i is: 2 Value of while loop i is: 3 Value of while loop i is: 4 Value of while loop i is: 5 Value of while loop i is: 6 Value of while loop i is: 7 Value of while loop i is: 8 Value of while loop i is: 9 Value of while loop i is: 10 While Loop with Continue/Break \u00b6 int idxWhileNew = 0 ; while ( idxWhileNew < 100 ) { idxWhileNew += 1 ; cout << \"Value of while loop i is: \" << idxWhile << endl ; if ( idxWhileNew == 10 ) { cout << \"Max value of 10 reached!\" << endl ; break ; } } Value of while loop i is: 10 Value of while loop i is: 10 Value of while loop i is: 10 Value of while loop i is: 10 Value of while loop i is: 10 Value of while loop i is: 10 Value of while loop i is: 10 Value of while loop i is: 10 Value of while loop i is: 10 Value of while loop i is: 10 Max value of 10 reached!","title":"C++"},{"location":"programming/cpp/cpp/#c","text":"Run Jupyter Notebook You can run the code for this section in this jupyter notebook link .","title":"C++"},{"location":"programming/cpp/cpp/#installation-of-interactive-c17","text":"Xeus-Cling is a game-changer where similar to Python Jupyter Notebooks, we can run C++ Jupyter Notebooks now. Run the following bash commands in sequence to create a C++ kernel for Jupyter Notebook. conda create -n cpp source activate cpp conda install -c conda-forge xeus-cling jupyter kernelspec install --user /home/ritchie/miniconda3/envs/cpp/share/jupyter/kernels/xcpp17 jupyter notebook","title":"Installation of Interactive C++17"},{"location":"programming/cpp/cpp/#printing","text":"","title":"Printing"},{"location":"programming/cpp/cpp/#printing-single-line","text":"// When you compile, the preprocessor runs and acts on all line with the pound key first // This is a preprocessor instruction that // essentially places the file with the name iostream into this spot #include <iostream> // Print one line std :: cout << \"Using Xeus Cling\" << std :: endl ; Using Xeus Cling","title":"Printing Single Line"},{"location":"programming/cpp/cpp/#printing-2-lines","text":"// Print two lines std :: cout << \"First Line \\n Second Line\" << std :: endl ; First Line Second Line","title":"Printing 2 Lines"},{"location":"programming/cpp/cpp/#printing-with-tabs","text":"// Print numbers with nicely formatted tabs std :: cout << \"One hundred: \\t \" ; std :: cout << ( float ) 1000 / 10 << std :: endl ; std :: cout << \"Two hundred: \\t \" ; std :: cout << ( double ) 2000 / 10 << std :: endl ; std :: cout << \"Three hundred: \\t \" ; std :: cout << ( double ) 3000 / 10 << std :: endl ; One hundred: 100 Two hundred: 200 Three hundred: 300","title":"Printing with Tabs"},{"location":"programming/cpp/cpp/#easier-printing-subjective-with-namespaces","text":"Gets irritating to use std in front of cout and endl to keep printing so we can use namespaces using namespace std ; cout << \"No need for messy std::\" << endl ; No need for messy std::","title":"Easier Printing (subjective) with Namespaces"},{"location":"programming/cpp/cpp/#variables","text":"","title":"Variables"},{"location":"programming/cpp/cpp/#general-memory-management","text":"In C++, you always need to determine each variable's type so the compiler will know how much memory (RAM) to allocate to the variable","title":"General memory management"},{"location":"programming/cpp/cpp/#types-of-variables","text":"Type Bytes (Size) Range of Values char 1 0 to 255 or -127 to 127 unsigned char 1 -127 to 127 signed char 1 0 to 255 bool 1 True or False int 4 -2,147,483,648 to 2,147,483,647 unsigned int 4 0 to 4,294,967,295 signed int 4 -2,147,483,648 to 2,147,483,647 short int 2 -32,768 to 32,767 long int 4 -2,147,483,648 to 2,147,483,647 float 4 3.4e-38 to 3.4e38 double 8 1.7e-308 to 1.7e-08 long double 8 1.7e-308 to 1.7e-08","title":"Types of Variables"},{"location":"programming/cpp/cpp/#integer-sizes","text":"Typical sizes Short integer: 2 bytes (will be smaller than long) Limitation is that the value of this short integer has a max value, you should be cautious of using short integers Long integer: 4 bytes Integer: 2 or 4 bytes Notes Technically these sizes can vary depending on your processor (32/64 bit) and compiler You should not assume the sizes but the hierarchy of sizes will not change (short memory < long memory)","title":"Integer sizes"},{"location":"programming/cpp/cpp/#integer-size","text":"cout << \"Size of an integer: \" << sizeof ( int ); Size of an integer: 4","title":"Integer size"},{"location":"programming/cpp/cpp/#short-integer-size","text":"cout << \"Size of a short integer: \" << sizeof ( short int ); Size of a short integer: 2","title":"Short integer size"},{"location":"programming/cpp/cpp/#long-integer-size","text":"cout << \"Size of an long integer: \" << sizeof ( long int ); Size of an long integer: 8","title":"Long integer size"},{"location":"programming/cpp/cpp/#unsigned-or-signed-integers","text":"Unsigned integers: only can hold positive integers Signed integers: can hold positive/negative integers","title":"Unsigned or signed integers"},{"location":"programming/cpp/cpp/#signed-short-integer","text":"cout << \"Size of an signed short integer: \" << sizeof ( signed short int ); Size of an signed short integer: 2","title":"Signed short integer"},{"location":"programming/cpp/cpp/#unsigned-short-integer","text":"cout << \"Size of an unsigned short integer: \" << sizeof ( unsigned short int ); Size of an unsigned short integer: 2","title":"Unsigned short integer"},{"location":"programming/cpp/cpp/#signed-long-integer","text":"cout << \"Size of an signed long integer: \" << sizeof ( signed long int ); Size of an signed long integer: 8","title":"Signed long integer"},{"location":"programming/cpp/cpp/#unsigned-long-integer","text":"cout << \"Size of an unsigned long integer: \" << sizeof ( unsigned long int ); Size of an unsigned long integer: 8","title":"Unsigned long integer"},{"location":"programming/cpp/cpp/#constants","text":"","title":"Constants"},{"location":"programming/cpp/cpp/#literal-constants","text":"int varOne = 20 ; std :: cout << varOne << std :: endl ; 20","title":"Literal Constants"},{"location":"programming/cpp/cpp/#enumerated-constants","text":"This enables you to create a new type! In this example we create a new type directions containing Up , Down , Left , and Right . enum directions { Up , Down , Left , Right }; directions goWhere ; goWhere = Right ; if ( goWhere == Right ) std :: cout << \"Go right\" << std :: endl ; Go right","title":"Enumerated Constants"},{"location":"programming/cpp/cpp/#functions","text":"","title":"Functions"},{"location":"programming/cpp/cpp/#function-without-return-value","text":"Syntax generally follows void FunctionName(argOne, argTwo) to define the function followed by FuntionName() to call the function. // Usage of void when the function does not return anything // In this example, this function prints out the multiplication result of two given numbers void MultiplyTwoNumbers ( int firstNum , int secondNum ) { // Define variable as integer type long int value ; value = firstNum * secondNum ; std :: cout << value << std :: endl ; }","title":"Function Without Return Value"},{"location":"programming/cpp/cpp/#multiply-two-numbers-3-and-2","text":"MultiplyTwoNumbers ( 3 , 2 ) 6","title":"Multiply Two Numbers 3 and 2"},{"location":"programming/cpp/cpp/#multiply-two-numbers-6-and-2","text":"MultiplyTwoNumbers ( 6 , 2 ) 12","title":"Multiply Two Numbers 6 and 2"},{"location":"programming/cpp/cpp/#aliases-function","text":"Say we want the variable value to be of type unsigned short int such that it's 2 bytes and can hold 2x the range of values compared to just short int . We can use typedef as an alias. Type Bytes (Size) Range of Values short int 2 -32,768 to 32,767 unsigned short int 2 0 to 65,536 // Usage of void when the function does not return anything // In this exmaple, this function prints out the multiplication result of two given numbers void MultiplyTwoNumbersWithAlias ( int firstNum , int secondNum ) { // Using an alias typedef unsigned short int ushortint ; // initializing value variable with ushortint type ushortint value ; value = firstNum * secondNum ; std :: cout << value << std :: endl ; }","title":"Aliases Function"},{"location":"programming/cpp/cpp/#multiply-two-numbers-10-and-10","text":"MultiplyTwoNumbersWithAlias ( 10 , 10 ) 100","title":"Multiply Two Numbers 10 and 10"},{"location":"programming/cpp/cpp/#multiply-two-numbers-1000-and-65","text":"MultiplyTwoNumbersWithAlias ( 1000 , 65 ) 65000","title":"Multiply Two Numbers 1000 and 65"},{"location":"programming/cpp/cpp/#multiply-two-numbers-1000-and-67","text":"Notice how you don't get 67,000? This is because our variable value of ushortint type can only hold values up to the integer 65,536. What this returns is the remainder of 67,000 - 65,536 = 1464 MultiplyTwoNumbersWithAlias ( 1000 , 67 ) 1464 std :: cout << 67 * 1000 - 65536 << std :: endl ; 1464","title":"Multiply Two Numbers 1000 and 67"},{"location":"programming/cpp/cpp/#function-with-return-value","text":"Unlike functions without return values where we use void to declare the function, here we use int to declare our function that returns values. // In this exmaple, this function returns the value of the multiplication of two numbers int MultiplyTwoNumbersNoPrint ( int firstNum , int secondNum ) { // Define variable as integer type long int value ; value = firstNum * secondNum ; return value ; }","title":"Function with Return Value"},{"location":"programming/cpp/cpp/#call-function","text":"// Declare variable with type long int returnValue ; // Call function returnValue = MultiplyTwoNumbersNoPrint ( 10 , 2 ); // Print variable std :: cout << returnValue << std :: endl ; 20","title":"Call Function"},{"location":"programming/cpp/cpp/#function-inner-workings","text":"Essentially our lines of codes translates to instruction pointers with unique memory addresses. Execution of instruction pointers operates on a \"LIFO\" basis, last in first out. Oversimplifying here, in our example, the last line is taken off first and it follows up // Code Space int varOneTest = 10 ; // Instruction pointer 100 std :: cout << varOneTest << std :: endl ; // Instruction Pointer 102 10 @0x7fa6b7de5460","title":"Function Inner Workings"},{"location":"programming/cpp/cpp/#arrays","text":"An array contains a sequence of elements with the same data type.","title":"Arrays"},{"location":"programming/cpp/cpp/#creating-an-array","text":"// This is how you declare an array of 50 elements each of type double double DoubleArray [ 50 ];","title":"Creating an Array"},{"location":"programming/cpp/cpp/#accessing-arrays-elements","text":"","title":"Accessing Array's Elements"},{"location":"programming/cpp/cpp/#first-element","text":"// This access the first array std :: cout << DoubleArray [ 0 ] << std :: endl ; 0","title":"First Element"},{"location":"programming/cpp/cpp/#last-element","text":"std :: cout << DoubleArray [ 49 ] << std :: endl ; 0","title":"Last Element"},{"location":"programming/cpp/cpp/#first-10-elements","text":"// In steps of 1 for ( int i = 0 ; i < 10 ; i ++ ) { // This is how you print a mix of characters and declared variables std :: cout << \"Element \" << i << \" contains \" << DoubleArray [ i ] << std :: endl ; } Element 0 contains 0 Element 1 contains 0 Element 2 contains 0 Element 3 contains 0 Element 4 contains 0 Element 5 contains 0 Element 6 contains 0 Element 7 contains 0 Element 8 contains 0 Element 9 contains 0 // In steps of 2 for ( int i = 0 ; i < 10 ; i += 2 ) { // This is how you print a mix of characters and declared variables std :: cout << \"Element \" << i << \" contains \" << DoubleArray [ i ] << std :: endl ; } Element 0 contains 0 Element 2 contains 0 Element 4 contains 0 Element 6 contains 0 Element 8 contains 0","title":"First 10 Elements"},{"location":"programming/cpp/cpp/#going-beyond-the-arrays-length","text":"This will return a warning that it's past the end of the array std :: cout << DoubleArray [ 50 ] << std :: endl ; input_line_36:2:15: warning: array index 50 is past the end of the array ( which contains 50 elements ) [ -Warray-bounds ] std::cout << DoubleArray[50] << std::endl; ^ ~~ input_line_32:3:1: note: array 'DoubleArray ' declared here double DoubleArray [ 50 ] ; ^ 4 .94066e-323","title":"Going Beyond The Array's Length"},{"location":"programming/cpp/cpp/#arrays-with-enumeration","text":"enum directionsNew { up , down , left , right , individualDirections }; int directionsArray [ individualDirections ] = { 1 , 2 , 3 , 4 }; std :: cout << \"Up value: \\t \" << directionsArray [ up ]; std :: cout << \" \\n Down value: \\t \" << directionsArray [ down ]; std :: cout << \" \\n Left value: \\t \" << directionsArray [ left ]; std :: cout << \" \\n Right value: \\t \" << directionsArray [ right ]; // This is the number of elements in the array std :: cout << \" \\n Num value: \\t \" << sizeof ( directionsArray ) / sizeof ( directionsArray [ 0 ]) << std :: endl ; Up value: 1 Down value: 2 Left value: 3 Right value: 4 Num value: 4","title":"Arrays with Enumeration"},{"location":"programming/cpp/cpp/#arrays-with-1-dimension-tensors","text":"","title":"Arrays with &gt;1 Dimension (Tensors)"},{"location":"programming/cpp/cpp/#multi-dimension-array-with-numbers","text":"// This is how you declare a multi-dimensional array of 5x5 elements each of type double double multiDimArray [ 5 ][ 5 ] = { { 1 , 2 , 3 , 4 , 5 }, { 2 , 2 , 3 , 4 , 5 }, { 3 , 2 , 3 , 4 , 5 }, { 4 , 2 , 3 , 4 , 5 }, { 5 , 2 , 3 , 4 , 5 } }; // Print each row of our 5x5 multi-dimensional array for ( int i = 0 ; i < 5 ; i ++ ) { for ( int j = 0 ; j < 5 ; j ++ ) { std :: cout << multiDimArray [ i ][ j ]; }; std :: cout << \" \\n \" << std :: endl ; }; 12345 22345 32345 42345 52345","title":"Multi Dimension Array with Numbers"},{"location":"programming/cpp/cpp/#multi-dimension-array-with-characters","text":"// This is how you declare a multi-dimensional array of 5x5 elements each of type char char multiDimArrayChars [ 5 ][ 5 ] = { { 'a' , 'b' , 'c' , 'd' , 'e' }, { 'b' , 'b' , 'c' , 'd' , 'e' }, { 'c' , 'b' , 'c' , 'd' , 'e' }, { 'd' , 'b' , 'c' , 'd' , 'e' }, { 'e' , 'b' , 'c' , 'd' , 'e' }, }; // Print each row of our 5x5 multi-dimensional array for ( int i = 0 ; i < 5 ; i ++ ) { for ( int j = 0 ; j < 5 ; j ++ ) { std :: cout << multiDimArrayChars [ i ][ j ]; }; std :: cout << \" \\n \" << std :: endl ; }; abcde bbcde cbcde dbcde ebcde","title":"Multi Dimension Array with Characters"},{"location":"programming/cpp/cpp/#copy-arrays","text":"","title":"Copy Arrays"},{"location":"programming/cpp/cpp/#copy-number-arrays","text":"double ArrayNumOne [] = { 1 , 2 , 3 , 4 , 5 }; double ArrayNumTwo [ 5 ]; // Use namespace std so it's cleaner using namespace std ; // Copy array with copy() copy ( begin ( ArrayNumOne ), end ( ArrayNumOne ), begin ( ArrayNumTwo )); // Print double type array with copy() copy ( begin ( ArrayNumTwo ), end ( ArrayNumTwo ), ostream_iterator < double > ( cout , \" \\n \" )); 1 2 3 4 5","title":"Copy Number Arrays"},{"location":"programming/cpp/cpp/#copy-string-arrays","text":"char ArrayCharOne [] = { 'a' , 'b' , 'c' , 'd' , 'e' }; char ArrayCharTwo [ 5 ]; // Use namespace std so it's cleaner using namespace std ; // Copy array with copy() copy ( begin ( ArrayCharOne ), end ( ArrayCharOne ), begin ( ArrayCharTwo )); // Print char type array with copy() copy ( begin ( ArrayCharTwo ), end ( ArrayCharTwo ), ostream_iterator < char > ( cout , \" \\n \" )); a b c d e","title":"Copy String Arrays"},{"location":"programming/cpp/cpp/#mathematical-operators","text":"Add, subtract, multiply, divide and modulus","title":"Mathematical Operators"},{"location":"programming/cpp/cpp/#add","text":"double addStatement = 5 + 5 ; std :: cout << addStatement ; 10","title":"Add"},{"location":"programming/cpp/cpp/#subtract","text":"double subtractStatement = 5 - 5 ; std :: cout << subtractStatement ; 0","title":"Subtract"},{"location":"programming/cpp/cpp/#divide","text":"double divideStatement = 5 / 5 ; std :: cout << divideStatement ; 1","title":"Divide"},{"location":"programming/cpp/cpp/#multiply","text":"double multiplyStatement = 5 * 5 ; std :: cout << multiplyStatement ; 25","title":"Multiply"},{"location":"programming/cpp/cpp/#modulus","text":"Gets the remainder of the division double modulusStatement = 8 % 5 ; std :: cout << modulusStatement ; 3","title":"Modulus"},{"location":"programming/cpp/cpp/#exponent","text":"Base to the power of something, this requires a new package called <cmath> that we want to include. #include <cmath> void SquareNumber ( int baseNum , int exponentNum ) { // Square the locally scoped variable with 2 int squaredNumber ; squaredNumber = pow ( baseNum , exponentNum ); std :: cout << \"Base of 2 with exponent of 2 gives: \" << squaredNumber << std :: endl ; } SquareNumber ( 2 , 2 ) Base of 2 with exponent of 2 gives: 4","title":"Exponent"},{"location":"programming/cpp/cpp/#incrementingdecrementing","text":"3 ways to do this, from least to most verbose","title":"Incrementing/Decrementing"},{"location":"programming/cpp/cpp/#methods","text":"","title":"Methods"},{"location":"programming/cpp/cpp/#method-1","text":"double idx = 1 ; idx ++ ; std :: cout << idx ; 2","title":"Method 1"},{"location":"programming/cpp/cpp/#method-2","text":"idx += 1 ; std :: cout << idx ; 3","title":"Method 2"},{"location":"programming/cpp/cpp/#method-3","text":"idx = idx + 1 ; std :: cout << idx ; 4","title":"Method 3"},{"location":"programming/cpp/cpp/#prefixpostfix","text":"","title":"Prefix/Postfix"},{"location":"programming/cpp/cpp/#prefix","text":"This will change both incremented variable and the new variable you assign the incremented variable to Summary: both variables will have the same values // Instantiate double a = 1 ; double b = 1 ; // Print original values cout << \"Old a: \\t \" << a << \" \\n \" ; cout << \"Old b: \\t \" << b << \" \\n \" ; // Prefix increment a = ++ b ; // Print new values cout << \"New a: \\t \" << a << \" \\n \" ; cout << \"New b: \\t \" << b << \" \\n \" ; Old a: 1 Old b: 1 New a: 2 New b: 2","title":"Prefix"},{"location":"programming/cpp/cpp/#postfix","text":"This will change only the incremented variable but not the variable it's assigned to Summary: incremented variable will change but not the variable it was assigned to // Instantiate double c = 2 ; double d = 2 ; // Print original values cout << \"Old c: \\t \" << c << \" \\n \" ; cout << \"Old d: \\t \" << d << \" \\n \" ; // Prefix increment c = d -- ; // Print new values, notice how only d decremented? c which is what d is assigned to doesn't change. cout << \"New c: \\t \" << c << \" \\n \" ; cout << \"New d: \\t \" << d << \" \\n \" ; Old c: 2 Old d: 2 New c: 2 New d: 1","title":"Postfix"},{"location":"programming/cpp/cpp/#conditional-statements","text":"","title":"Conditional Statements"},{"location":"programming/cpp/cpp/#if","text":"int maxValue = 10 ; // Increment till 10 for ( int i = 0 ; i <= 10 ; i += 2 ) { // Stop if the number reaches 10 (inclusive) if ( i == maxValue ) { cout << \"Reached max value!\" ; cout << \" \\n Value is \" << i << endl ; }; }; Reached max value! Value is 10","title":"If"},{"location":"programming/cpp/cpp/#else","text":"int newMaxValue = 20 ; // Increment till 10 for ( int i = 0 ; i <= 10 ; i += 2 ) { // Stop if the number reaches 10 (inclusive) if ( i == newMaxValue ) { cout << \"Reached max value!\" ; cout << \" \\n Value is \" << i << endl ; } // Else print current value else { cout << \" \\n Current Value is \" << i << endl ; } } Current Value is 0 Current Value is 2 Current Value is 4 Current Value is 6 Current Value is 8 Current Value is 10","title":"Else"},{"location":"programming/cpp/cpp/#logical-operators","text":"","title":"Logical Operators"},{"location":"programming/cpp/cpp/#and","text":"int varOneNew = 10 ; int varTwo = 10 ; int varCheckOne = 10 ; int varCheckTwo = 5 ; // This should print out if (( varOneNew == varCheckOne ) && ( varTwo == varCheckOne )) { std :: cout << \"Both values equal to 10!\" << std :: endl ; } // This should not print out as varTwo does not equal to 5 if (( varOneNew == varCheckOne ) && ( varTwo == varCheckTwo )) { std :: cout << \"VarOneNew equals to 10, VarTwo equals to 5\" << std :: endl ; } Both values equal to 10!","title":"And"},{"location":"programming/cpp/cpp/#or","text":"// On the contrary, this exact same statement would print out // as VarOne is equal to 10 and we are using an OR operator if (( varOneNew == varCheckOne ) || ( varTwo == varCheckTwo )) { std :: cout << \"VarOneNew equals to 10 or VarTwo equals to 5\" << std :: endl ; } VarOneNew equals to 10 or VarTwo equals to 5","title":"Or"},{"location":"programming/cpp/cpp/#not","text":"// This would print out as VarTwo is not equal to 5 if ( varTwo != varCheckTwo ) { std :: cout << \"VarTwo (10) is not equal to VarCheckTwo (5).\" << std :: endl ; } VarTwo (10) is not equal to VarCheckTwo (5).","title":"Not"},{"location":"programming/cpp/cpp/#getting-user-input","text":"using namespace std ; long double inputOne , inputTwo ; cout << \"This program multiplies 2 given numbers \\n \" ; cout << \"Enter first number: \\n \" ; cin >> inputOne ; cout << \"Enter second number: \\n \" ; cin >> inputTwo ; cout << \"Multiplication value: \" << inputOne * inputTwo << endl ; This program multiplies 2 given numbers Enter first number: 10 Enter second number: 10 Multiplication value: 100","title":"Getting User Input"},{"location":"programming/cpp/cpp/#loops","text":"","title":"Loops"},{"location":"programming/cpp/cpp/#for-loop","text":"for ( int i = 0 ; i < 10 ; i += 1 ) { cout << \"Value of i is: \" << i << endl ; } Value of i is: 0 Value of i is: 1 Value of i is: 2 Value of i is: 3 Value of i is: 4 Value of i is: 5 Value of i is: 6 Value of i is: 7 Value of i is: 8 Value of i is: 9","title":"For Loop"},{"location":"programming/cpp/cpp/#while-loop","text":"int idxWhile = 0 ; while ( idxWhile < 10 ) { idxWhile += 1 ; cout << \"Value of while loop i is: \" << idxWhile << endl ; } Value of while loop i is: 1 Value of while loop i is: 2 Value of while loop i is: 3 Value of while loop i is: 4 Value of while loop i is: 5 Value of while loop i is: 6 Value of while loop i is: 7 Value of while loop i is: 8 Value of while loop i is: 9 Value of while loop i is: 10","title":"While Loop"},{"location":"programming/cpp/cpp/#while-loop-with-continuebreak","text":"int idxWhileNew = 0 ; while ( idxWhileNew < 100 ) { idxWhileNew += 1 ; cout << \"Value of while loop i is: \" << idxWhile << endl ; if ( idxWhileNew == 10 ) { cout << \"Max value of 10 reached!\" << endl ; break ; } } Value of while loop i is: 10 Value of while loop i is: 10 Value of while loop i is: 10 Value of while loop i is: 10 Value of while loop i is: 10 Value of while loop i is: 10 Value of while loop i is: 10 Value of while loop i is: 10 Value of while loop i is: 10 Value of while loop i is: 10 Max value of 10 reached!","title":"While Loop with Continue/Break"},{"location":"programming/electron/electron/","text":"Electron \u00b6 Why Electron \u00b6 We choose to cover electron as you can easily use it as a front-end application across any platforms (Windows, MacOS, Linux or even a mobile application) for your AI applications. Installation of Electron \u00b6 npm i -D electron@latest Creating Electron Project \u00b6 Critical Files \u00b6 You should have 3 base files package.json , main.js and index.html to have a basic application. mkdir app cd app npm init touch main.js touch index.html Edit package.json \u00b6 When you run npm init , it should create a package.json file. But we need to make some tiny changes to leverage on electron. Key fields name : name of your app, can be anything version : version of your app, can be anything main : main javascript file, we recommend using main.js scripts : here you want to copy the whole scripts section to leverage on electron devDependencies : electron version required { \"name\" : \"dlw\" , \"version\" : \"0.1.0\" , \"main\" : \"main.js\" , \"scripts\" : { \"start\" : \"electron .\" }, \"devDependencies\" : { \"electron\" : \"^6.0.8\" } } Edit main.js \u00b6 This beautiful boilerplate code is provided by Electron, full credits to them. const { app , BrowserWindow } = require ( 'electron' ) // Keep a global reference of the window object, if you don't, the window will // be closed automatically when the JavaScript object is garbage collected. let win function createWindow () { // Create the browser window. win = new BrowserWindow ({ width : 800 , height : 600 , webPreferences : { nodeIntegration : true } }) // and load the index.html of the app. win . loadFile ( 'index.html' ) // Open the DevTools. // win.webContents.openDevTools() // Emitted when the window is closed. win . on ( 'closed' , () => { // Dereference the window object, usually you would store windows // in an array if your app supports multi windows, this is the time // when you should delete the corresponding element. win = null }) } // This method will be called when Electron has finished // initialization and is ready to create browser windows. // Some APIs can only be used after this event occurs. app . on ( 'ready' , createWindow ) // Quit when all windows are closed. app . on ( 'window-all-closed' , () => { // On macOS it is common for applications and their menu bar // to stay active until the user quits explicitly with Cmd + Q if ( process . platform !== 'darwin' ) { app . quit () } }) app . on ( 'activate' , () => { // On macOS it's common to re-create a window in the app when the // dock icon is clicked and there are no other windows open. if ( win === null ) { createWindow () } }) // In this file you can include the rest of your app's specific main process // code. You can also put them in separate files and require them here. Edit index.html \u00b6 I modified this script from electron's boilerplate code where it will display critical dependencies' versions for your node, chrome and electron. <!DOCTYPE html> < html > < head > < meta charset = \"UTF-8\" > < title > Dashboard </ title > </ head > < body > < h1 > Dashboard </ h1 > < h2 > Environment </ h2 > < br /> Node: < script > document . write ( process . versions . node )</ script > < br /> Chrome: < script > document . write ( process . versions . chrome )</ script > < br /> Electron: < script > document . write ( process . versions . electron )</ script > </ body > </ html > Starting App \u00b6 This will start your electron application. npm start Packaging Electron App \u00b6 Wine \u00b6 The reason for installing Wine is being able to package Electron applications for the Windows platform, creating the executable file app.exe like any other application on Windows. The final aim of our tutorial is to package the app for Windows, MacOS and Ubuntu. Installation of Wine \u00b6 This assumes installation on Ubuntu 16.04 xenial , if you're on Ubuntu 18.04 or 19.04, change to bionic and disco respectively. Also, this works on 64-bit system architecture. cd ~ wget -qO - https://dl.winehq.org/wine-builds/winehq.key | sudo apt-key add - sudo apt-add-repository 'deb https://dl.winehq.org/wine-builds/ubuntu/ xenial main' sudo apt-get update sudo apt-get install --install-recommends winehq-stable sudo chown root:root ~/.wine Check Wine Version \u00b6 wine --version Packaging Windows Application \u00b6 This packages the application churning the necessary files and the executable app.exe for windows 64 bit. electron-packager ./app app --platform = win32 --arch = x64 Python Scripts \u00b6 Installing Python Node Package \u00b6 So we want to easily create Python scripts and run through Javascript in the Electron application. This can be done via python-shell npm package. sudo npm install --save python-shell Creating \"Hello from JS\" \u00b6 Javascript \u00b6 In your main.js file, you would want to add the following code. This leverages on the python-shell package to send a message to hello_world.py and receive the message subsequently. // Start Python shell let { PythonShell } = require ( 'python-shell' ) // Start shell for specific script for communicating let pyshell = new PythonShell ( './scripts/hello_world.py' ); // Send a message to the Python script via stdin pyshell . send ( 'Hello from JS' ); // Receive message from Python script pyshell . on ( 'message' , function ( message ) { console . log ( message ); }); // End the input stream and allow the process to exit pyshell . end ( function ( err , code , signal ) { if ( err ) throw err ; // console.log('The exit code was: ' + code); // console.log('The exit signal was: ' + signal); console . log ( 'finished' ); }); Python \u00b6 Create a folder scripts to hold all your Python scripts. Then create a Python file named hello_world.py with the following content. import sys msg_from_js = sys . stdin . read () print ( msg_from_js ) Run App \u00b6 Run via npm start and you'll see this in your bash output. Viola! We managed to call hello_world.py via main.js through the python-shell package. Next task, we will be passing this message to index.html . Hello from JS finished","title":"Electron"},{"location":"programming/electron/electron/#electron","text":"","title":"Electron"},{"location":"programming/electron/electron/#why-electron","text":"We choose to cover electron as you can easily use it as a front-end application across any platforms (Windows, MacOS, Linux or even a mobile application) for your AI applications.","title":"Why Electron"},{"location":"programming/electron/electron/#installation-of-electron","text":"npm i -D electron@latest","title":"Installation of Electron"},{"location":"programming/electron/electron/#creating-electron-project","text":"","title":"Creating Electron Project"},{"location":"programming/electron/electron/#critical-files","text":"You should have 3 base files package.json , main.js and index.html to have a basic application. mkdir app cd app npm init touch main.js touch index.html","title":"Critical Files"},{"location":"programming/electron/electron/#edit-packagejson","text":"When you run npm init , it should create a package.json file. But we need to make some tiny changes to leverage on electron. Key fields name : name of your app, can be anything version : version of your app, can be anything main : main javascript file, we recommend using main.js scripts : here you want to copy the whole scripts section to leverage on electron devDependencies : electron version required { \"name\" : \"dlw\" , \"version\" : \"0.1.0\" , \"main\" : \"main.js\" , \"scripts\" : { \"start\" : \"electron .\" }, \"devDependencies\" : { \"electron\" : \"^6.0.8\" } }","title":"Edit package.json"},{"location":"programming/electron/electron/#edit-mainjs","text":"This beautiful boilerplate code is provided by Electron, full credits to them. const { app , BrowserWindow } = require ( 'electron' ) // Keep a global reference of the window object, if you don't, the window will // be closed automatically when the JavaScript object is garbage collected. let win function createWindow () { // Create the browser window. win = new BrowserWindow ({ width : 800 , height : 600 , webPreferences : { nodeIntegration : true } }) // and load the index.html of the app. win . loadFile ( 'index.html' ) // Open the DevTools. // win.webContents.openDevTools() // Emitted when the window is closed. win . on ( 'closed' , () => { // Dereference the window object, usually you would store windows // in an array if your app supports multi windows, this is the time // when you should delete the corresponding element. win = null }) } // This method will be called when Electron has finished // initialization and is ready to create browser windows. // Some APIs can only be used after this event occurs. app . on ( 'ready' , createWindow ) // Quit when all windows are closed. app . on ( 'window-all-closed' , () => { // On macOS it is common for applications and their menu bar // to stay active until the user quits explicitly with Cmd + Q if ( process . platform !== 'darwin' ) { app . quit () } }) app . on ( 'activate' , () => { // On macOS it's common to re-create a window in the app when the // dock icon is clicked and there are no other windows open. if ( win === null ) { createWindow () } }) // In this file you can include the rest of your app's specific main process // code. You can also put them in separate files and require them here.","title":"Edit main.js"},{"location":"programming/electron/electron/#edit-indexhtml","text":"I modified this script from electron's boilerplate code where it will display critical dependencies' versions for your node, chrome and electron. <!DOCTYPE html> < html > < head > < meta charset = \"UTF-8\" > < title > Dashboard </ title > </ head > < body > < h1 > Dashboard </ h1 > < h2 > Environment </ h2 > < br /> Node: < script > document . write ( process . versions . node )</ script > < br /> Chrome: < script > document . write ( process . versions . chrome )</ script > < br /> Electron: < script > document . write ( process . versions . electron )</ script > </ body > </ html >","title":"Edit index.html"},{"location":"programming/electron/electron/#starting-app","text":"This will start your electron application. npm start","title":"Starting App"},{"location":"programming/electron/electron/#packaging-electron-app","text":"","title":"Packaging Electron App"},{"location":"programming/electron/electron/#wine","text":"The reason for installing Wine is being able to package Electron applications for the Windows platform, creating the executable file app.exe like any other application on Windows. The final aim of our tutorial is to package the app for Windows, MacOS and Ubuntu.","title":"Wine"},{"location":"programming/electron/electron/#installation-of-wine","text":"This assumes installation on Ubuntu 16.04 xenial , if you're on Ubuntu 18.04 or 19.04, change to bionic and disco respectively. Also, this works on 64-bit system architecture. cd ~ wget -qO - https://dl.winehq.org/wine-builds/winehq.key | sudo apt-key add - sudo apt-add-repository 'deb https://dl.winehq.org/wine-builds/ubuntu/ xenial main' sudo apt-get update sudo apt-get install --install-recommends winehq-stable sudo chown root:root ~/.wine","title":"Installation of Wine"},{"location":"programming/electron/electron/#check-wine-version","text":"wine --version","title":"Check Wine Version"},{"location":"programming/electron/electron/#packaging-windows-application","text":"This packages the application churning the necessary files and the executable app.exe for windows 64 bit. electron-packager ./app app --platform = win32 --arch = x64","title":"Packaging Windows Application"},{"location":"programming/electron/electron/#python-scripts","text":"","title":"Python Scripts"},{"location":"programming/electron/electron/#installing-python-node-package","text":"So we want to easily create Python scripts and run through Javascript in the Electron application. This can be done via python-shell npm package. sudo npm install --save python-shell","title":"Installing Python Node Package"},{"location":"programming/electron/electron/#creating-hello-from-js","text":"","title":"Creating \"Hello from JS\""},{"location":"programming/electron/electron/#javascript","text":"In your main.js file, you would want to add the following code. This leverages on the python-shell package to send a message to hello_world.py and receive the message subsequently. // Start Python shell let { PythonShell } = require ( 'python-shell' ) // Start shell for specific script for communicating let pyshell = new PythonShell ( './scripts/hello_world.py' ); // Send a message to the Python script via stdin pyshell . send ( 'Hello from JS' ); // Receive message from Python script pyshell . on ( 'message' , function ( message ) { console . log ( message ); }); // End the input stream and allow the process to exit pyshell . end ( function ( err , code , signal ) { if ( err ) throw err ; // console.log('The exit code was: ' + code); // console.log('The exit signal was: ' + signal); console . log ( 'finished' ); });","title":"Javascript"},{"location":"programming/electron/electron/#python","text":"Create a folder scripts to hold all your Python scripts. Then create a Python file named hello_world.py with the following content. import sys msg_from_js = sys . stdin . read () print ( msg_from_js )","title":"Python"},{"location":"programming/electron/electron/#run-app","text":"Run via npm start and you'll see this in your bash output. Viola! We managed to call hello_world.py via main.js through the python-shell package. Next task, we will be passing this message to index.html . Hello from JS finished","title":"Run App"},{"location":"programming/javascript/javascript/","text":"Javascript \u00b6 Run Jupyter Notebook You can run the code for this section in this jupyter notebook link . Installation of iJavascript \u00b6 This will enable the Javascript kernel to be installed in your jupyter notebook kernel list so you can play with Javascript easily. For now, the installation requires the ancient Python 2.7. Hopefully this changes in the future. conda create -n py27 python=2.7 conda activate py27 sudo npm install -g --unsafe-perm ijavascript conda install jupyter ijsinstall jupyter notebook Variables & Constants \u00b6 Variable \u00b6 Declaring Variable \u00b6 var randomNumber = 1.52 Read out Variable \u00b6 // Check value randomNumber 1.52 Read out via Console \u00b6 // Using the console via log and error console . log ( randomNumber ) console . error ( randomNumber ) 1.52 1.52 Constants \u00b6 Declaring Constant \u00b6 This is a weird one, because in Javascript when you declare a constant, you are essentially unable to overwrite the constant subsequently. It's useful when you want a fixed value. const cannotOverwriteNumber = 2.22 Read out Constant \u00b6 cannotOverwriteNumber 2.22 Overwrite Constant (Error) \u00b6 This will throw an error because this constant has been declared once and you cannot do it again. const cannotOverwriteNumber = 1.52 evalmachine.<anonymous>:1 const cannotOverwriteNumber = 1.52 ^ SyntaxError: Identifier 'cannotOverwriteNumber' has already been declared at evalmachine.<anonymous>:1:1 at Script.runInThisContext (vm.js:122:20) at Object.runInThisContext (vm.js:329:38) at run ([eval]:1054:15) at onRunRequest ([eval]:888:18) at onMessage ([eval]:848:13) at process.emit (events.js:198:13) at emit (internal/child_process.js:832:12) at process._tickCallback (internal/process/next_tick.js:63:19)","title":"Javascript"},{"location":"programming/javascript/javascript/#javascript","text":"Run Jupyter Notebook You can run the code for this section in this jupyter notebook link .","title":"Javascript"},{"location":"programming/javascript/javascript/#installation-of-ijavascript","text":"This will enable the Javascript kernel to be installed in your jupyter notebook kernel list so you can play with Javascript easily. For now, the installation requires the ancient Python 2.7. Hopefully this changes in the future. conda create -n py27 python=2.7 conda activate py27 sudo npm install -g --unsafe-perm ijavascript conda install jupyter ijsinstall jupyter notebook","title":"Installation of iJavascript"},{"location":"programming/javascript/javascript/#variables-constants","text":"","title":"Variables &amp; Constants"},{"location":"programming/javascript/javascript/#variable","text":"","title":"Variable"},{"location":"programming/javascript/javascript/#declaring-variable","text":"var randomNumber = 1.52","title":"Declaring Variable"},{"location":"programming/javascript/javascript/#read-out-variable","text":"// Check value randomNumber 1.52","title":"Read out Variable"},{"location":"programming/javascript/javascript/#read-out-via-console","text":"// Using the console via log and error console . log ( randomNumber ) console . error ( randomNumber ) 1.52 1.52","title":"Read out via Console"},{"location":"programming/javascript/javascript/#constants","text":"","title":"Constants"},{"location":"programming/javascript/javascript/#declaring-constant","text":"This is a weird one, because in Javascript when you declare a constant, you are essentially unable to overwrite the constant subsequently. It's useful when you want a fixed value. const cannotOverwriteNumber = 2.22","title":"Declaring Constant"},{"location":"programming/javascript/javascript/#read-out-constant","text":"cannotOverwriteNumber 2.22","title":"Read out Constant"},{"location":"programming/javascript/javascript/#overwrite-constant-error","text":"This will throw an error because this constant has been declared once and you cannot do it again. const cannotOverwriteNumber = 1.52 evalmachine.<anonymous>:1 const cannotOverwriteNumber = 1.52 ^ SyntaxError: Identifier 'cannotOverwriteNumber' has already been declared at evalmachine.<anonymous>:1:1 at Script.runInThisContext (vm.js:122:20) at Object.runInThisContext (vm.js:329:38) at run ([eval]:1054:15) at onRunRequest ([eval]:888:18) at onMessage ([eval]:848:13) at process.emit (events.js:198:13) at emit (internal/child_process.js:832:12) at process._tickCallback (internal/process/next_tick.js:63:19)","title":"Overwrite Constant (Error)"},{"location":"programming/numpycupy/linalg_numpy_cupy/","text":"Linear Algebra with NumPy and CuPy \u00b6 In this section, we will be covering linear algebra and using numpy for CPU-based matrix manipulation and cupy for GPU-based matrix manipulation. Types of Matrices \u00b6 M x N Matrix M rows and N columns 1 x N Matrix (Row Vector) 1 row and N columns M x 1 Matrix (Column Vector) M rows and 1 column 1 x 1 Matrix (Scalar) 1 row and 1 column N x M Matrix from M x N Matrix (Transposed Matrix) Swapping M rows to columns and N columns to rows where matrix \\(A\\) then becomes transposed matrix \\(A\\prime\\) Symmetric Matrix M x N Matrix is equal to N x M Transposed Matrix M x M Matrix (Square Matrix) M rows = N columns Diagonal Matrix all matrix elements are zeroes except for the diagonal Identity matrix all matrix elements are zeroes except for the diagonals being 1's Multiplication and Subtraction \u00b6 Element-wise Matrix Addition/Subtraction add/subtract every individual element \\(a_{ij}\\) of the first matrix \\(A_{M \\times N}\\) with \\(b_{ij}\\) of the second matrix \\(B_{M \\times N}\\) Scalar Multiplication with Matrix multiply scalar \\(\\lambda\\) with every individual element \\(a_{ij}\\) of the matrix \\(A_{M \\times N}\\) Matrix Multiplication can only be done where \\(A_{M \\times N}\\) and \\(B_{N \\times O}\\) have a coinciding dimension \\(N\\) which is the same to yield a new matrix \\(C_{N \\times O}\\) of \\(N \\times O\\) dimension new element \\(c_{ij} = \\sum _{q=1} ^N a_{iq} b_{qj}\\) essentially the first row and first column element of the new matrix is equal to the summation element-wise multiplication of the first row of matrix \\(A\\) with the first column of matrix \\(B\\) Determinant of Square Matrix \u00b6 Determinant of 2 x 2 Square Matrix \u00b6 The determinant of a 2x2 square matrix can be derived by multiplying all the elements on the main diagonal and subtracting the multiplication of all the elements in the other diagonal which will result in a scalar value. \\[ A = \\begin{bmatrix}a & b\\\\c & d\\end{bmatrix} \\] \\[\\det(A) = | A | = ad - bc \\] Determinant of 3 x 3 Square Matrix \u00b6 \\[ \\begin{bmatrix} a & b &c \\\\ d& e &f \\\\ g& h &i \\end{bmatrix} \\\\ = a (-1)^{(1+1)}\\det \\begin{bmatrix} e & f\\\\ h & i \\end{bmatrix} + b (-1)^{(1+2)} \\det \\begin{bmatrix} d & f\\\\ g & i \\end{bmatrix} + c (-1)^{(1+3)} \\det \\begin{bmatrix} d & e\\\\ g & h \\end{bmatrix} \\] Determinant Types \u00b6 singular matrix: \\(\\det(A) = 0\\) where at least 2 rows/columns are linearly dependent non-singular: \\(\\det(A) \\neq 0\\) where no rows/columns are linearly dependent (matrix has full rank) Inverse of a Non-Singular Square Matrix \u00b6 Unique Property \u00b6 The inverse can be calculated only if it is a non-singular square matrix hence it is useful to first calculate the determinant. Also the inverse of the matrix yields a useful and unique property where \\[ AA^{-1} = A^{-1}A = I \\] Inverse and Identity of 2 x 2 Non-Singular Square Matrix \u00b6 \\[ A=\\begin{bmatrix}a & b \\\\c & d \\end{bmatrix} \\] \\[ A^{-1}=\\frac{1}{ad-bc}\\begin{bmatrix}d & -b \\\\-c & a \\end{bmatrix} \\] \\[ \\begin{array}{lcl}AA^{-1}&=&\\begin{bmatrix}a & b \\\\c & d \\end{bmatrix}\\frac{1}{ad-bc}\\begin{bmatrix}d & -b \\\\-c & a \\end{bmatrix}\\\\ &=&\\frac{1}{ad-bc}\\begin{bmatrix}ad+b(-c) & a(-b)+b(a) \\\\cd+d(-c) & c(-b)+d(a) \\end{bmatrix}\\\\ &=&\\begin{bmatrix}1 & 0\\\\0 & 1 \\end{bmatrix}\\end{array} \\] Linear System of Equations in Matrices \u00b6 We can express many linear equations in the form of matrices for example \\(AX= B\\) Where we have \\(A\\) representing our parameters, \\(B\\) representing our input variables and \\(B\\) representing our constant/bias variables. \\[ A = \\begin{bmatrix} a_{11} & a_{12} & \\cdots & a_{1N} \\\\ a_{21} & a_{22} & \\cdots & a_{2N} \\\\ \\vdots & \\vdots & \\ddots & \\vdots \\\\ a_{M1} & a_{M2} & \\cdots & a_{MN} \\end{bmatrix}, X = \\begin{bmatrix} x_1 \\\\ x_2 \\\\ \\vdots \\\\ x_M \\end{bmatrix}, B = \\begin{bmatrix} b_1 \\\\ b_2 \\\\ \\vdots \\\\ b_M \\end{bmatrix} \\] If we find \\(\\det (A)\\) to be non-zero where the matrix is a non-singular matrix, the system of equations would have a unique solution. Hence, given \\(A\\) and \\(B\\) , we can find out the unique solution in \\(B\\) . \\[ AX = B \\\\ A^{-1}AX = A^{-1}B \\\\ IX = A^{-1}B \\\\ X = A^{-1}B \\] Solving System of Equations for Square and Non-Singular \u00b6 If the matrix A is square and \\(det(A) \\neq 0\\) (non-singular, inverse matrix can be calculated), then Cramer's rule can be applied to solve the linear system of equations. Process for a 2 x 2 Non-Singular Matrix A \u00b6 Replace First Column of \\(A\\) with \\(B\\) to get \\(A_1\\) $$ x_1 = \\frac{\\det(A_1)}{\\det(A)} $$ Replace Second Column of \\(A\\) with \\(B\\) to get \\(A_2\\) $$ x_2 = \\frac{\\det(A_2)}{\\det(A)} $$ Repeat till all columns covered for bigger matrices Eigenvalue (characteristic root) and eigenvalue (characteristic value) \u00b6 Given a square matrix \\(A\\) , eigenvalue \\(\\lambda\\) and eigenvector \\(v\\) where \\(v \\neq 0\\) , we have: \\[ A v = \\lambda v \\\\ A v = \\lambda I v \\\\ Av - \\lambda I v = 0 \\\\ (A - \\lambda I)v = 0 \\\\ \\] Since \\(v \\neq 0\\) then we have characteristic matrix \\((A - \\lambda I) = 0\\) . Solving eigens \u00b6 We can solve for the determinant of the characteristic matrix (characteristic polynomial) \\(|A - \\lambda I| = 0\\) through this characteristic equation. We will get multiple values of \\(\\lambda\\) Substitute \\(\\lambda\\) into \\((A - \\lambda I)v = 0\\) , solve for \\(v\\) If infinite solution (no constant values for x and y), impose uniqueness with \\(v\\prime v = 1\\) in this case \\(\\begin{bmatrix} x & y \\end{bmatrix} \\begin{bmatrix} x \\\\ y \\end{bmatrix} = x^2 + y^2 = 1\\) Substitute y into \\(v\\prime v = 1\\) to solve for x, then solve for y Substitute x and y to solve for \\(v\\) Simple eigenvector \\(\\lambda _1 \\neq \\lambda _2\\) Repeated/double eigenvector \\(\\lambda _1 = \\lambda _2\\) Properties of eigenvalue and eigenvector \u00b6 If \\(|A - \\lambda I| = 0\\) , singular therefore infinite solutions, hence: \\(\\lambda \\gt 0\\) : positive definite \\(\\lambda \\ge 0\\) : positive semi-definite \\(\\lambda \\lt 0\\) : negative definite \\(\\lambda \\le 0\\) : negative semi-definite","title":"NumPy and CuPy"},{"location":"programming/numpycupy/linalg_numpy_cupy/#linear-algebra-with-numpy-and-cupy","text":"In this section, we will be covering linear algebra and using numpy for CPU-based matrix manipulation and cupy for GPU-based matrix manipulation.","title":"Linear Algebra with NumPy and CuPy"},{"location":"programming/numpycupy/linalg_numpy_cupy/#types-of-matrices","text":"M x N Matrix M rows and N columns 1 x N Matrix (Row Vector) 1 row and N columns M x 1 Matrix (Column Vector) M rows and 1 column 1 x 1 Matrix (Scalar) 1 row and 1 column N x M Matrix from M x N Matrix (Transposed Matrix) Swapping M rows to columns and N columns to rows where matrix \\(A\\) then becomes transposed matrix \\(A\\prime\\) Symmetric Matrix M x N Matrix is equal to N x M Transposed Matrix M x M Matrix (Square Matrix) M rows = N columns Diagonal Matrix all matrix elements are zeroes except for the diagonal Identity matrix all matrix elements are zeroes except for the diagonals being 1's","title":"Types of Matrices"},{"location":"programming/numpycupy/linalg_numpy_cupy/#multiplication-and-subtraction","text":"Element-wise Matrix Addition/Subtraction add/subtract every individual element \\(a_{ij}\\) of the first matrix \\(A_{M \\times N}\\) with \\(b_{ij}\\) of the second matrix \\(B_{M \\times N}\\) Scalar Multiplication with Matrix multiply scalar \\(\\lambda\\) with every individual element \\(a_{ij}\\) of the matrix \\(A_{M \\times N}\\) Matrix Multiplication can only be done where \\(A_{M \\times N}\\) and \\(B_{N \\times O}\\) have a coinciding dimension \\(N\\) which is the same to yield a new matrix \\(C_{N \\times O}\\) of \\(N \\times O\\) dimension new element \\(c_{ij} = \\sum _{q=1} ^N a_{iq} b_{qj}\\) essentially the first row and first column element of the new matrix is equal to the summation element-wise multiplication of the first row of matrix \\(A\\) with the first column of matrix \\(B\\)","title":"Multiplication and Subtraction"},{"location":"programming/numpycupy/linalg_numpy_cupy/#determinant-of-square-matrix","text":"","title":"Determinant of Square Matrix"},{"location":"programming/numpycupy/linalg_numpy_cupy/#determinant-of-2-x-2-square-matrix","text":"The determinant of a 2x2 square matrix can be derived by multiplying all the elements on the main diagonal and subtracting the multiplication of all the elements in the other diagonal which will result in a scalar value. \\[ A = \\begin{bmatrix}a & b\\\\c & d\\end{bmatrix} \\] \\[\\det(A) = | A | = ad - bc \\]","title":"Determinant of 2 x 2 Square Matrix"},{"location":"programming/numpycupy/linalg_numpy_cupy/#determinant-of-3-x-3-square-matrix","text":"\\[ \\begin{bmatrix} a & b &c \\\\ d& e &f \\\\ g& h &i \\end{bmatrix} \\\\ = a (-1)^{(1+1)}\\det \\begin{bmatrix} e & f\\\\ h & i \\end{bmatrix} + b (-1)^{(1+2)} \\det \\begin{bmatrix} d & f\\\\ g & i \\end{bmatrix} + c (-1)^{(1+3)} \\det \\begin{bmatrix} d & e\\\\ g & h \\end{bmatrix} \\]","title":"Determinant of 3 x 3 Square Matrix"},{"location":"programming/numpycupy/linalg_numpy_cupy/#determinant-types","text":"singular matrix: \\(\\det(A) = 0\\) where at least 2 rows/columns are linearly dependent non-singular: \\(\\det(A) \\neq 0\\) where no rows/columns are linearly dependent (matrix has full rank)","title":"Determinant Types"},{"location":"programming/numpycupy/linalg_numpy_cupy/#inverse-of-a-non-singular-square-matrix","text":"","title":"Inverse of a Non-Singular Square Matrix"},{"location":"programming/numpycupy/linalg_numpy_cupy/#unique-property","text":"The inverse can be calculated only if it is a non-singular square matrix hence it is useful to first calculate the determinant. Also the inverse of the matrix yields a useful and unique property where \\[ AA^{-1} = A^{-1}A = I \\]","title":"Unique Property"},{"location":"programming/numpycupy/linalg_numpy_cupy/#inverse-and-identity-of-2-x-2-non-singular-square-matrix","text":"\\[ A=\\begin{bmatrix}a & b \\\\c & d \\end{bmatrix} \\] \\[ A^{-1}=\\frac{1}{ad-bc}\\begin{bmatrix}d & -b \\\\-c & a \\end{bmatrix} \\] \\[ \\begin{array}{lcl}AA^{-1}&=&\\begin{bmatrix}a & b \\\\c & d \\end{bmatrix}\\frac{1}{ad-bc}\\begin{bmatrix}d & -b \\\\-c & a \\end{bmatrix}\\\\ &=&\\frac{1}{ad-bc}\\begin{bmatrix}ad+b(-c) & a(-b)+b(a) \\\\cd+d(-c) & c(-b)+d(a) \\end{bmatrix}\\\\ &=&\\begin{bmatrix}1 & 0\\\\0 & 1 \\end{bmatrix}\\end{array} \\]","title":"Inverse and Identity of 2 x 2 Non-Singular Square Matrix"},{"location":"programming/numpycupy/linalg_numpy_cupy/#linear-system-of-equations-in-matrices","text":"We can express many linear equations in the form of matrices for example \\(AX= B\\) Where we have \\(A\\) representing our parameters, \\(B\\) representing our input variables and \\(B\\) representing our constant/bias variables. \\[ A = \\begin{bmatrix} a_{11} & a_{12} & \\cdots & a_{1N} \\\\ a_{21} & a_{22} & \\cdots & a_{2N} \\\\ \\vdots & \\vdots & \\ddots & \\vdots \\\\ a_{M1} & a_{M2} & \\cdots & a_{MN} \\end{bmatrix}, X = \\begin{bmatrix} x_1 \\\\ x_2 \\\\ \\vdots \\\\ x_M \\end{bmatrix}, B = \\begin{bmatrix} b_1 \\\\ b_2 \\\\ \\vdots \\\\ b_M \\end{bmatrix} \\] If we find \\(\\det (A)\\) to be non-zero where the matrix is a non-singular matrix, the system of equations would have a unique solution. Hence, given \\(A\\) and \\(B\\) , we can find out the unique solution in \\(B\\) . \\[ AX = B \\\\ A^{-1}AX = A^{-1}B \\\\ IX = A^{-1}B \\\\ X = A^{-1}B \\]","title":"Linear System of Equations in Matrices"},{"location":"programming/numpycupy/linalg_numpy_cupy/#solving-system-of-equations-for-square-and-non-singular","text":"If the matrix A is square and \\(det(A) \\neq 0\\) (non-singular, inverse matrix can be calculated), then Cramer's rule can be applied to solve the linear system of equations.","title":"Solving System of Equations for Square and Non-Singular"},{"location":"programming/numpycupy/linalg_numpy_cupy/#process-for-a-2-x-2-non-singular-matrix-a","text":"Replace First Column of \\(A\\) with \\(B\\) to get \\(A_1\\) $$ x_1 = \\frac{\\det(A_1)}{\\det(A)} $$ Replace Second Column of \\(A\\) with \\(B\\) to get \\(A_2\\) $$ x_2 = \\frac{\\det(A_2)}{\\det(A)} $$ Repeat till all columns covered for bigger matrices","title":"Process for a 2 x 2 Non-Singular Matrix A"},{"location":"programming/numpycupy/linalg_numpy_cupy/#eigenvalue-characteristic-root-and-eigenvalue-characteristic-value","text":"Given a square matrix \\(A\\) , eigenvalue \\(\\lambda\\) and eigenvector \\(v\\) where \\(v \\neq 0\\) , we have: \\[ A v = \\lambda v \\\\ A v = \\lambda I v \\\\ Av - \\lambda I v = 0 \\\\ (A - \\lambda I)v = 0 \\\\ \\] Since \\(v \\neq 0\\) then we have characteristic matrix \\((A - \\lambda I) = 0\\) .","title":"Eigenvalue (characteristic root) and eigenvalue (characteristic value)"},{"location":"programming/numpycupy/linalg_numpy_cupy/#solving-eigens","text":"We can solve for the determinant of the characteristic matrix (characteristic polynomial) \\(|A - \\lambda I| = 0\\) through this characteristic equation. We will get multiple values of \\(\\lambda\\) Substitute \\(\\lambda\\) into \\((A - \\lambda I)v = 0\\) , solve for \\(v\\) If infinite solution (no constant values for x and y), impose uniqueness with \\(v\\prime v = 1\\) in this case \\(\\begin{bmatrix} x & y \\end{bmatrix} \\begin{bmatrix} x \\\\ y \\end{bmatrix} = x^2 + y^2 = 1\\) Substitute y into \\(v\\prime v = 1\\) to solve for x, then solve for y Substitute x and y to solve for \\(v\\) Simple eigenvector \\(\\lambda _1 \\neq \\lambda _2\\) Repeated/double eigenvector \\(\\lambda _1 = \\lambda _2\\)","title":"Solving eigens"},{"location":"programming/numpycupy/linalg_numpy_cupy/#properties-of-eigenvalue-and-eigenvector","text":"If \\(|A - \\lambda I| = 0\\) , singular therefore infinite solutions, hence: \\(\\lambda \\gt 0\\) : positive definite \\(\\lambda \\ge 0\\) : positive semi-definite \\(\\lambda \\lt 0\\) : negative definite \\(\\lambda \\le 0\\) : negative semi-definite","title":"Properties of eigenvalue and eigenvector"},{"location":"programming/plotting/intro/","text":"Plotting \u00b6 Packages you will learn to use Fast professional static plots: ggplot2 and rpy2 Complicated static plots: matplotlib Live stream and/or interactive plots: plotly Dashboard: streamlit Typically with tremendous mastery of matplotlib, you will be able to create any good looking plots static or interactive. However, it takes many more lines of code and is relatively complicated. Thus, before we go there, we will be covering the fastest way to create professional looking static plots with rpy2 and ggplot rapidly. It is unconventional but it yields the fastest result for time-sensitive learners. We'll then move backwards to then create more complicated plots with matplotlib. Subsequently we'll create live stream and interactive plots with plotly. Finally we will be bringing everything together with streamlit as a dashboard. As usual, you can go freestyle and just use any of our open-source material you think is useful and you do not need to follow the sequence we are suggesting.","title":"Plotting"},{"location":"programming/plotting/intro/#plotting","text":"Packages you will learn to use Fast professional static plots: ggplot2 and rpy2 Complicated static plots: matplotlib Live stream and/or interactive plots: plotly Dashboard: streamlit Typically with tremendous mastery of matplotlib, you will be able to create any good looking plots static or interactive. However, it takes many more lines of code and is relatively complicated. Thus, before we go there, we will be covering the fastest way to create professional looking static plots with rpy2 and ggplot rapidly. It is unconventional but it yields the fastest result for time-sensitive learners. We'll then move backwards to then create more complicated plots with matplotlib. Subsequently we'll create live stream and interactive plots with plotly. Finally we will be bringing everything together with streamlit as a dashboard. As usual, you can go freestyle and just use any of our open-source material you think is useful and you do not need to follow the sequence we are suggesting.","title":"Plotting"},{"location":"programming/python/python/","text":"Python \u00b6 Lists \u00b6 Creating List: Manual Fill \u00b6 lst = [ 0 , 1 , 2 , 3 ] print ( lst ) [0, 1, 2, 3] Creating List: List Comprehension \u00b6 lst = [ i for i in range ( 4 )] print ( lst ) [0, 1, 2, 3] Joining List with Blanks \u00b6 # To use .join(), your list needs to be of type string lst_to_string = list ( map ( str , lst )) # Join the list of strings lst_join = ' ' . join ( lst_to_string ) print ( lst_join ) 0 1 2 3 Joining List with Comma \u00b6 # Join the list of strings lst_join = ', ' . join ( lst_to_string ) print ( lst_join ) 0, 1, 2, 3 Checking Lists Equal: Method 1 \u00b6 Returns True if equal, and False if unequal lst_unequal = [ 1 , 1 , 2 , 3 , 4 , 4 ] lst_equal = [ 0 , 0 , 0 , 0 , 0 , 0 ] print ( '-' * 50 ) print ( 'Unequal List' ) print ( '-' * 50 ) print ( lst_unequal [ 1 :]) print ( lst_unequal [: - 1 ]) bool_equal = lst_unequal [ 1 :] == lst_unequal [: - 1 ] print ( bool_equal ) print ( '-' * 50 ) print ( 'Equal List' ) print ( '-' * 50 ) print ( lst_equal [ 1 :]) print ( lst_equal [: - 1 ]) bool_equal = lst_equal [ 1 :] == lst_equal [: - 1 ] print ( bool_equal ) -------------------------------------------------- Unequal List -------------------------------------------------- [1, 2, 3, 4, 4] [1, 1, 2, 3, 4] False -------------------------------------------------- Equal List -------------------------------------------------- [0, 0, 0, 0, 0] [0, 0, 0, 0, 0] True Checking Lists Equal: Method 2 \u00b6 Returns True if equal, and False if unequal. Here, all essentially checks that there is no False in the list. print ( '-' * 50 ) print ( 'Unequal List' ) print ( '-' * 50 ) lst_check = [ i == lst_unequal [ 0 ] for i in lst_unequal ] bool_equal = all ( lst_check ) print ( bool_equal ) print ( '-' * 50 ) print ( 'Equal List' ) print ( '-' * 50 ) lst_check = [ i == lst_equal [ 0 ] for i in lst_equal ] bool_equal = all ( lst_check ) print ( bool_equal ) -------------------------------------------------- Unequal List -------------------------------------------------- False -------------------------------------------------- Equal List -------------------------------------------------- True Sets \u00b6 Removing Duplicate from List \u00b6 Sets can be very useful for quickly removing duplicates from a list, essentially finding unique values lst_one = [ 1 , 2 , 3 , 5 ] lst_two = [ 1 , 1 , 2 , 4 ] lst_both = lst_one + lst_two lst_no_duplicate = list ( set ( lst_both )) print ( f 'Original Combined List { lst_both } ' ) print ( f 'No Duplicated Combined List { lst_no_duplicate } ' ) Original Combined List [1, 2, 3, 5, 1, 1, 2, 4] No Duplicated Combined List [1, 2, 3, 4, 5] Lambda, map, filter, reduce, partial \u00b6 Lambda \u00b6 The syntax is simple lambda your_variables: your_operation Add Function \u00b6 add = lambda x , y : x + y add ( 2 , 3 ) 5 Multiply Function \u00b6 multiply = lambda x , y : x * y multiply ( 2 , 3 ) 6 Map \u00b6 Create List \u00b6 lst = [ i for i in range ( 11 )] print ( lst ) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] Map Square Function to List \u00b6 square_element = map ( lambda x : x ** 2 , lst ) # This gives you a map object print ( square_element ) # You need to explicitly return a list print ( list ( square_element )) <map object at 0x7f08c8620438> [0, 1, 4, 9, 16, 25, 36, 49, 64, 81, 100] Create Multiple List \u00b6 lst_1 = [ 1 , 2 , 3 , 4 ] lst_2 = [ 2 , 4 , 6 , 8 ] lst_3 = [ 3 , 6 , 9 , 12 ] Map Add Function to Multiple Lists \u00b6 add_elements = map ( lambda x , y , z : x + y + z , lst_1 , lst_2 , lst_3 ) print ( list ( add_elements )) [6, 12, 18, 24] Filter \u00b6 Create List \u00b6 lst = [ i for i in range ( 10 )] print ( lst ) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] Filter multiples of 3 \u00b6 multiples_of_three = filter ( lambda x : x % 3 == 0 , lst ) print ( list ( multiples_of_three )) [0, 3, 6, 9] Reduce \u00b6 The syntax is reduce(function, sequence) . The function is applied to the elements in the list in a sequential manner. Meaning if lst = [1, 2, 3, 4] and you have a sum function, you would arrive with ((1+2) + 3) + 4 . from functools import reduce sum_all = reduce ( lambda x , y : x + y , lst ) # Here we've 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 print ( sum_all ) print ( 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 ) 45 45 Partial \u00b6 Allows us to predefine and freeze a function's argument. Combined with lambda, it allows us to have more flexibility beyond lambda's restriction of a single line. from functools import partial def display_sum_three ( a , b , c ): sum_all = a + b + c print ( f 'Sum is { sum_all } ' ) fixed_args_func = partial ( display_sum_three , b = 3 , c = 4 ) # Given fixed arguments b=3 and c=4 # We add the new variable against the fixed arguments var_int = 1 fixed_args_func ( var_int ) # More advanced mapping with partial # Add a variable from 0 to 9 to the constants print ( '-' * 50 ) _ = list ( map ( fixed_args_func , list ( range ( 10 )))) # How about using with lambda to modifying constants without # declaring your function again? print ( '-' * 50 ) _ = list ( map ( lambda x : fixed_args_func ( x , b = 2 ), list ( range ( 10 )))) Sum is 8 -------------------------------------------------- Sum is 7 Sum is 8 Sum is 9 Sum is 10 Sum is 11 Sum is 12 Sum is 13 Sum is 14 Sum is 15 Sum is 16 -------------------------------------------------- Sum is 6 Sum is 7 Sum is 8 Sum is 9 Sum is 10 Sum is 11 Sum is 12 Sum is 13 Sum is 14 Sum is 15 Generators \u00b6 Why: generators are typically more memory-efficient than using simple for loops Imagine wanting to sum digits 0 to 1 trillion, using a list containing those numbers and summing them would be very RAM memory-inefficient. Using a generator would allow you to sum one digit sequentially, staggering the RAM memory usage in steps. What: generator basically a function that returns an iterable object where we can iterate one bye one Types: generator functions and generator expressions Dependencies: we need to install a memory profiler, so install via pip install memory_profiler Simple custom generator function example: sum 1 to 1,000,000 \u00b6 What: let's create a simple generator, allowing us to iterate through the digits 1 to 1,000,000 (inclusive) one by one with an increment of 1 at each step and summing them How: 2 step process with a while and a yield # Load memory profiler % load_ext memory_profiler # Here we take a step from 1 def create_numbers ( end_number ): current_number = 1 # Step 1: while while current_number <= end_number : # Step 2: yield yield current_number # Add to current number current_number += 1 # Here we sum the digits 1 to 100 (inclusive) and time it % memit total = sum ( create_numbers ( 1e6 )) print ( total ) peak memory: 46.50 MiB, increment: 0.28 MiB 500000500000 Without generator function: sum with list \u00b6 Say we don't use a generator, and have a list of digits 0 to 1,000,000 (inclusive) in memory then sum them. Notice how this is double the memory than using a generator! % memit total = sum ( list ( range ( int ( 1e6 ) + 1 ))) print ( total ) peak memory: 85.14 MiB, increment: 38.38 MiB 500000500000 Without generator function: sum with for loop \u00b6 Say we don't use a generator and don't put all our numbers into a list Notiice how this is much better than summing a list but still worst than a generator in terms of memory? def sum_with_loop ( end_number ): total = 0 for i in range ( end_number + 1 ): i += 1 total += i return total % memit total = sum_with_loop ( int ( 1e6 )) print ( total ) peak memory: 54.49 MiB, increment: 0.00 MiB 500001500001 Generator expression \u00b6 Like list/dictionary expressions, we can have generator expressions too We can quickly create generators this way, allowing us to make computations on the fly rather than pre-compute on a whole list/array of numbers This is more memory efficient # Define the list list_of_numbers = list ( range ( 10 )) # Find square root using the list comprehension list_of_results = [ number ** 2 for number in list_of_numbers ] print ( list_of_results ) # Use generator expression to calculate the square root generator_of_results = ( number ** 2 for number in list_of_numbers ) print ( generator_of_results ) for idx in range ( 10 ): print ( next ( generator_of_results )) [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] <generator object <genexpr> at 0x7f08c85aa4f8> 0 1 4 9 16 25 36 49 64 81 Decorators \u00b6 This allows us to to modify our original function or even entirely replace it without changing the function's code. It sounds mind-boggling, but a simple case I would like to illustrate here is using decorators for consistent logging (formatted print statements). For us to understand decorators, we'll first need to understand: first class objects *args *kwargs First Class Objects \u00b6 def outer (): def inner (): print ( 'Inside inner() function.' ) # This returns a function. return inner # Here, we are assigning `outer()` function to the object `call_outer`. call_outer = outer () # Then we call `call_outer()` call_outer () Inside inner() function. *args \u00b6 This is used to indicate that positional arguments should be stored in the variable args * is for iterables and positional parameters # Define dummy function def dummy_func ( * args ): print ( args ) # * allows us to extract positional variables from an iterable when we are calling a function dummy_func ( * range ( 10 )) # If we do not use *, this would happen dummy_func ( range ( 10 )) # See how we can have varying arguments? dummy_func ( * range ( 2 )) (0, 1, 2, 3, 4, 5, 6, 7, 8, 9) (range(0, 10),) (0, 1) **kwargs \u00b6 ** is for dictionaries & key/value pairs # New dummy function def dummy_func_new ( ** kwargs ): print ( kwargs ) # Call function with no arguments dummy_func_new () # Call function with 2 arguments dummy_func_new ( a = 0 , b = 1 ) # Again, there's no limit to the number of arguments. dummy_func_new ( a = 0 , b = 1 , c = 2 ) # Or we can just pass the whole dictionary object if we want new_dict = { 'a' : 0 , 'b' : 1 , 'c' : 2 , 'd' : 3 } dummy_func_new ( ** new_dict ) {} {'a': 0, 'b': 1} {'a': 0, 'b': 1, 'c': 2} {'a': 0, 'b': 1, 'c': 2, 'd': 3} Decorators as Logger and Debugging \u00b6 A simple way to remember the power of decorators is that the decorator (the nested function illustrated below) can (1) access the passed arguments of the decorated function and (2) access the decorated function Therefore this allows us to modify the decorated function without changing the decorated function # Create a nested function that will be our decorator def function_inspector ( func ): def inner ( * args , ** kwargs ): result = func ( * args , ** kwargs ) print ( f 'Function args: { args } ' ) print ( f 'Function kwargs: { kwargs } ' ) print ( f 'Function return result: { result } ' ) return result return inner # Decorate our multiply function with our logger for easy logging # Of arguments pass to the function and results returned @function_inspector def multiply_func ( num_one , num_two ): return num_one * num_two multiply_result = multiply_func ( num_one = 1 , num_two = 2 ) Function args: () Function kwargs: {'num_one': 1, 'num_two': 2} Function return result: 2 Dates \u00b6 Get Current Date \u00b6 import datetime now = datetime . datetime . now () print ( now ) 2019-08-12 14:20:45.604849 Get Clean String Current Date \u00b6 # YYYY-MM-DD now . date () . strftime ( '20%y-%m- %d ' ) '2019-08-12' Count Business Days \u00b6 # Number of business days in a month from Jan 2019 to Feb 2019 import numpy as np days = np . busday_count ( '2019-01' , '2019-02' ) print ( days ) 23 Progress Bars \u00b6 TQDM \u00b6 Simple progress bar via pip install tqdm from tqdm import tqdm import time for i in tqdm ( range ( 100 )): time . sleep ( 0.1 ) pass 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 100/100 [00:10<00:00, 9.91it/s] Check Paths \u00b6 Check Path Exists \u00b6 Check if directory exists import os directory = 'new_dir' print ( os . path . exists ( directory )) # Magic function to list all folders ! ls - d */ False ls: cannot access '*/': No such file or directory Check Path Exists Otherwise Create Folder \u00b6 Check if directory exists, otherwise make folder if not os . path . exists ( directory ): os . makedirs ( directory ) # Magic function to list all folders ! ls - d */ # Remove directory ! rmdir new_dir new_dir/ Exception Handling \u00b6 Try, Except, Finally: Error \u00b6 This is very handy and often exploited to patch up (save) poorly written code You can use general exceptions or specific ones like ValueError , KeyboardInterrupt and MemoryError to name a few value_one = 'a' value_two = 2 # Try the following line of code try : final_sum = value_one / value_two print ( 'Code passed!' ) # If the code above fails, code nested under except will be executed except : print ( 'Code failed!' ) # This will run no matter whether the nested code in try or except is executed finally : print ( 'Ran code block regardless of error or not.' ) Code failed! Ran code block regardless of error or not. Try, Except, Finally: No Error \u00b6 There won't be errors because you can divide 4 with 2 value_one = 4 value_two = 2 # Try the following line of code try : final_sum = value_one / value_two print ( 'Code passed!' ) # If the code above fails, code nested under except will be executed except : print ( 'Code failed!' ) # This will run no matter whether the nested code in try or except is executed finally : print ( 'Ran code block regardless of error or not.' ) Code passed! Ran code block regardless of error or not. Assertion \u00b6 This comes in handy when you want to enforce strict requirmenets of a certain value, shape, value type, or others for i in range ( 10 ): assert i <= 5 , 'Value is more than 5, rejected' print ( f 'Passed assertion for value { i } ' ) Passed assertion for value 0 Passed assertion for value 1 Passed assertion for value 2 Passed assertion for value 3 Passed assertion for value 4 Passed assertion for value 5 --------------------------------------------------------------------------- AssertionError Traceback (most recent call last) <ipython-input-2-d9d077e139a9> in <module> 1 for i in range(10): ----> 2 assert i <= 5, 'Value is more than 5, rejected' 3 print(f'Passed assertion for value {i}') AssertionError: Value is more than 5, rejected Asynchronous \u00b6 Concurrency, Parallelism, Asynchronous \u00b6 Concurrency (single CPU core): multiple threads on a single core running in sequence , only 1 thread is making progress at any point Think of 1 human, packing a box then wrapping the box Parallelism (mutliple GPU cores): multiple threads on multiple cores running in parallel , multiple threads can be making progress Think of 2 humans, one packing a box, another wrapping the box Asynchronous: concurrency but with a more dynamic system that moves amongst threads more efficiently rather than waiting for a task to finish then moving to the next task Python's asyncio allows us to code asynchronously Benefits: Scales better if you need to wait on a lot of processes Less memory (easier in this sense) to wait on thousands of co-routines than running on thousands of threads Good for IO bound uses like reading/saving from databases while subsequently running other computation Easier management than multi-thread processing like in parallel programming In the sense that everything operates sequentially in the same memory space Asynchronous Key Components \u00b6 The three main parts are (1) coroutines and subroutines, (2) event loops, and (3) future. Co-routine and subroutines Subroutine: the usual function Coroutine: this allows us to maintain states with memory of where things stopped so we can swap amongst subroutines async declares a function as a coroutine await to call a coroutine Event loops Future Synchronous 2 Function Calls \u00b6 import timeit def add_numbers ( num_1 , num_2 ): print ( 'Adding' ) time . sleep ( 1 ) return num_1 + num_2 def display_sum ( num_1 , num_2 ): total_sum = add_numbers ( num_1 , num_2 ) print ( f 'Total sum { total_sum } ' ) def main (): display_sum ( 2 , 2 ) display_sum ( 2 , 2 ) start = timeit . default_timer () main () end = timeit . default_timer () total_time = end - start print ( f 'Total time { total_time : .2f } s' ) Adding Total sum 4 Adding Total sum 4 Total time 2.00s Parallel 2 Function Calls \u00b6 from multiprocessing import Pool from functools import partial start = timeit . default_timer () pool = Pool () result = pool . map ( partial ( display_sum , num_2 = 2 ), [ 2 , 2 ]) end = timeit . default_timer () total_time = end - start print ( f 'Total time { total_time : .2f } s' ) Adding Adding Total sum 4 Total sum 4 Total time 1.08s Asynchronous 2 Function Calls \u00b6 For this use case, it'll take half the time compared to a synchronous application and slightly faster than parallel application (although not always true for parallel except in this case) import asyncio import timeit import time async def add_numbers ( num_1 , num_2 ): print ( 'Adding' ) await asyncio . sleep ( 1 ) return num_1 + num_2 async def display_sum ( num_1 , num_2 ): total_sum = await add_numbers ( num_1 , num_2 ) print ( f 'Total sum { total_sum } ' ) async def main (): # .gather allows us to group subroutines await asyncio . gather ( display_sum ( 2 , 2 ), display_sum ( 2 , 2 )) start = timeit . default_timer () # For .ipynb, event loop already done await main () # For .py # asyncio.run(main()) end = timeit . default_timer () total_time = end - start print ( f 'Total time { total_time : .4f } s' ) Adding Adding Total sum 4 Total sum 4 Total time 1.0021s","title":"Python"},{"location":"programming/python/python/#python","text":"","title":"Python"},{"location":"programming/python/python/#lists","text":"","title":"Lists"},{"location":"programming/python/python/#creating-list-manual-fill","text":"lst = [ 0 , 1 , 2 , 3 ] print ( lst ) [0, 1, 2, 3]","title":"Creating List: Manual Fill"},{"location":"programming/python/python/#creating-list-list-comprehension","text":"lst = [ i for i in range ( 4 )] print ( lst ) [0, 1, 2, 3]","title":"Creating List: List Comprehension"},{"location":"programming/python/python/#joining-list-with-blanks","text":"# To use .join(), your list needs to be of type string lst_to_string = list ( map ( str , lst )) # Join the list of strings lst_join = ' ' . join ( lst_to_string ) print ( lst_join ) 0 1 2 3","title":"Joining List with Blanks"},{"location":"programming/python/python/#joining-list-with-comma","text":"# Join the list of strings lst_join = ', ' . join ( lst_to_string ) print ( lst_join ) 0, 1, 2, 3","title":"Joining List with Comma"},{"location":"programming/python/python/#checking-lists-equal-method-1","text":"Returns True if equal, and False if unequal lst_unequal = [ 1 , 1 , 2 , 3 , 4 , 4 ] lst_equal = [ 0 , 0 , 0 , 0 , 0 , 0 ] print ( '-' * 50 ) print ( 'Unequal List' ) print ( '-' * 50 ) print ( lst_unequal [ 1 :]) print ( lst_unequal [: - 1 ]) bool_equal = lst_unequal [ 1 :] == lst_unequal [: - 1 ] print ( bool_equal ) print ( '-' * 50 ) print ( 'Equal List' ) print ( '-' * 50 ) print ( lst_equal [ 1 :]) print ( lst_equal [: - 1 ]) bool_equal = lst_equal [ 1 :] == lst_equal [: - 1 ] print ( bool_equal ) -------------------------------------------------- Unequal List -------------------------------------------------- [1, 2, 3, 4, 4] [1, 1, 2, 3, 4] False -------------------------------------------------- Equal List -------------------------------------------------- [0, 0, 0, 0, 0] [0, 0, 0, 0, 0] True","title":"Checking Lists Equal: Method 1"},{"location":"programming/python/python/#checking-lists-equal-method-2","text":"Returns True if equal, and False if unequal. Here, all essentially checks that there is no False in the list. print ( '-' * 50 ) print ( 'Unequal List' ) print ( '-' * 50 ) lst_check = [ i == lst_unequal [ 0 ] for i in lst_unequal ] bool_equal = all ( lst_check ) print ( bool_equal ) print ( '-' * 50 ) print ( 'Equal List' ) print ( '-' * 50 ) lst_check = [ i == lst_equal [ 0 ] for i in lst_equal ] bool_equal = all ( lst_check ) print ( bool_equal ) -------------------------------------------------- Unequal List -------------------------------------------------- False -------------------------------------------------- Equal List -------------------------------------------------- True","title":"Checking Lists Equal: Method 2"},{"location":"programming/python/python/#sets","text":"","title":"Sets"},{"location":"programming/python/python/#removing-duplicate-from-list","text":"Sets can be very useful for quickly removing duplicates from a list, essentially finding unique values lst_one = [ 1 , 2 , 3 , 5 ] lst_two = [ 1 , 1 , 2 , 4 ] lst_both = lst_one + lst_two lst_no_duplicate = list ( set ( lst_both )) print ( f 'Original Combined List { lst_both } ' ) print ( f 'No Duplicated Combined List { lst_no_duplicate } ' ) Original Combined List [1, 2, 3, 5, 1, 1, 2, 4] No Duplicated Combined List [1, 2, 3, 4, 5]","title":"Removing Duplicate from List"},{"location":"programming/python/python/#lambda-map-filter-reduce-partial","text":"","title":"Lambda, map, filter, reduce, partial"},{"location":"programming/python/python/#lambda","text":"The syntax is simple lambda your_variables: your_operation","title":"Lambda"},{"location":"programming/python/python/#add-function","text":"add = lambda x , y : x + y add ( 2 , 3 ) 5","title":"Add Function"},{"location":"programming/python/python/#multiply-function","text":"multiply = lambda x , y : x * y multiply ( 2 , 3 ) 6","title":"Multiply Function"},{"location":"programming/python/python/#map","text":"","title":"Map"},{"location":"programming/python/python/#create-list","text":"lst = [ i for i in range ( 11 )] print ( lst ) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]","title":"Create List"},{"location":"programming/python/python/#map-square-function-to-list","text":"square_element = map ( lambda x : x ** 2 , lst ) # This gives you a map object print ( square_element ) # You need to explicitly return a list print ( list ( square_element )) <map object at 0x7f08c8620438> [0, 1, 4, 9, 16, 25, 36, 49, 64, 81, 100]","title":"Map Square Function to List"},{"location":"programming/python/python/#create-multiple-list","text":"lst_1 = [ 1 , 2 , 3 , 4 ] lst_2 = [ 2 , 4 , 6 , 8 ] lst_3 = [ 3 , 6 , 9 , 12 ]","title":"Create Multiple List"},{"location":"programming/python/python/#map-add-function-to-multiple-lists","text":"add_elements = map ( lambda x , y , z : x + y + z , lst_1 , lst_2 , lst_3 ) print ( list ( add_elements )) [6, 12, 18, 24]","title":"Map Add Function to Multiple Lists"},{"location":"programming/python/python/#filter","text":"","title":"Filter"},{"location":"programming/python/python/#create-list_1","text":"lst = [ i for i in range ( 10 )] print ( lst ) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]","title":"Create List"},{"location":"programming/python/python/#filter-multiples-of-3","text":"multiples_of_three = filter ( lambda x : x % 3 == 0 , lst ) print ( list ( multiples_of_three )) [0, 3, 6, 9]","title":"Filter multiples of 3"},{"location":"programming/python/python/#reduce","text":"The syntax is reduce(function, sequence) . The function is applied to the elements in the list in a sequential manner. Meaning if lst = [1, 2, 3, 4] and you have a sum function, you would arrive with ((1+2) + 3) + 4 . from functools import reduce sum_all = reduce ( lambda x , y : x + y , lst ) # Here we've 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 print ( sum_all ) print ( 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 ) 45 45","title":"Reduce"},{"location":"programming/python/python/#partial","text":"Allows us to predefine and freeze a function's argument. Combined with lambda, it allows us to have more flexibility beyond lambda's restriction of a single line. from functools import partial def display_sum_three ( a , b , c ): sum_all = a + b + c print ( f 'Sum is { sum_all } ' ) fixed_args_func = partial ( display_sum_three , b = 3 , c = 4 ) # Given fixed arguments b=3 and c=4 # We add the new variable against the fixed arguments var_int = 1 fixed_args_func ( var_int ) # More advanced mapping with partial # Add a variable from 0 to 9 to the constants print ( '-' * 50 ) _ = list ( map ( fixed_args_func , list ( range ( 10 )))) # How about using with lambda to modifying constants without # declaring your function again? print ( '-' * 50 ) _ = list ( map ( lambda x : fixed_args_func ( x , b = 2 ), list ( range ( 10 )))) Sum is 8 -------------------------------------------------- Sum is 7 Sum is 8 Sum is 9 Sum is 10 Sum is 11 Sum is 12 Sum is 13 Sum is 14 Sum is 15 Sum is 16 -------------------------------------------------- Sum is 6 Sum is 7 Sum is 8 Sum is 9 Sum is 10 Sum is 11 Sum is 12 Sum is 13 Sum is 14 Sum is 15","title":"Partial"},{"location":"programming/python/python/#generators","text":"Why: generators are typically more memory-efficient than using simple for loops Imagine wanting to sum digits 0 to 1 trillion, using a list containing those numbers and summing them would be very RAM memory-inefficient. Using a generator would allow you to sum one digit sequentially, staggering the RAM memory usage in steps. What: generator basically a function that returns an iterable object where we can iterate one bye one Types: generator functions and generator expressions Dependencies: we need to install a memory profiler, so install via pip install memory_profiler","title":"Generators"},{"location":"programming/python/python/#simple-custom-generator-function-example-sum-1-to-1000000","text":"What: let's create a simple generator, allowing us to iterate through the digits 1 to 1,000,000 (inclusive) one by one with an increment of 1 at each step and summing them How: 2 step process with a while and a yield # Load memory profiler % load_ext memory_profiler # Here we take a step from 1 def create_numbers ( end_number ): current_number = 1 # Step 1: while while current_number <= end_number : # Step 2: yield yield current_number # Add to current number current_number += 1 # Here we sum the digits 1 to 100 (inclusive) and time it % memit total = sum ( create_numbers ( 1e6 )) print ( total ) peak memory: 46.50 MiB, increment: 0.28 MiB 500000500000","title":"Simple custom generator function example: sum 1 to 1,000,000"},{"location":"programming/python/python/#without-generator-function-sum-with-list","text":"Say we don't use a generator, and have a list of digits 0 to 1,000,000 (inclusive) in memory then sum them. Notice how this is double the memory than using a generator! % memit total = sum ( list ( range ( int ( 1e6 ) + 1 ))) print ( total ) peak memory: 85.14 MiB, increment: 38.38 MiB 500000500000","title":"Without generator function: sum with list"},{"location":"programming/python/python/#without-generator-function-sum-with-for-loop","text":"Say we don't use a generator and don't put all our numbers into a list Notiice how this is much better than summing a list but still worst than a generator in terms of memory? def sum_with_loop ( end_number ): total = 0 for i in range ( end_number + 1 ): i += 1 total += i return total % memit total = sum_with_loop ( int ( 1e6 )) print ( total ) peak memory: 54.49 MiB, increment: 0.00 MiB 500001500001","title":"Without generator function: sum with for loop"},{"location":"programming/python/python/#generator-expression","text":"Like list/dictionary expressions, we can have generator expressions too We can quickly create generators this way, allowing us to make computations on the fly rather than pre-compute on a whole list/array of numbers This is more memory efficient # Define the list list_of_numbers = list ( range ( 10 )) # Find square root using the list comprehension list_of_results = [ number ** 2 for number in list_of_numbers ] print ( list_of_results ) # Use generator expression to calculate the square root generator_of_results = ( number ** 2 for number in list_of_numbers ) print ( generator_of_results ) for idx in range ( 10 ): print ( next ( generator_of_results )) [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] <generator object <genexpr> at 0x7f08c85aa4f8> 0 1 4 9 16 25 36 49 64 81","title":"Generator expression"},{"location":"programming/python/python/#decorators","text":"This allows us to to modify our original function or even entirely replace it without changing the function's code. It sounds mind-boggling, but a simple case I would like to illustrate here is using decorators for consistent logging (formatted print statements). For us to understand decorators, we'll first need to understand: first class objects *args *kwargs","title":"Decorators"},{"location":"programming/python/python/#first-class-objects","text":"def outer (): def inner (): print ( 'Inside inner() function.' ) # This returns a function. return inner # Here, we are assigning `outer()` function to the object `call_outer`. call_outer = outer () # Then we call `call_outer()` call_outer () Inside inner() function.","title":"First Class Objects"},{"location":"programming/python/python/#args","text":"This is used to indicate that positional arguments should be stored in the variable args * is for iterables and positional parameters # Define dummy function def dummy_func ( * args ): print ( args ) # * allows us to extract positional variables from an iterable when we are calling a function dummy_func ( * range ( 10 )) # If we do not use *, this would happen dummy_func ( range ( 10 )) # See how we can have varying arguments? dummy_func ( * range ( 2 )) (0, 1, 2, 3, 4, 5, 6, 7, 8, 9) (range(0, 10),) (0, 1)","title":"*args"},{"location":"programming/python/python/#kwargs","text":"** is for dictionaries & key/value pairs # New dummy function def dummy_func_new ( ** kwargs ): print ( kwargs ) # Call function with no arguments dummy_func_new () # Call function with 2 arguments dummy_func_new ( a = 0 , b = 1 ) # Again, there's no limit to the number of arguments. dummy_func_new ( a = 0 , b = 1 , c = 2 ) # Or we can just pass the whole dictionary object if we want new_dict = { 'a' : 0 , 'b' : 1 , 'c' : 2 , 'd' : 3 } dummy_func_new ( ** new_dict ) {} {'a': 0, 'b': 1} {'a': 0, 'b': 1, 'c': 2} {'a': 0, 'b': 1, 'c': 2, 'd': 3}","title":"**kwargs"},{"location":"programming/python/python/#decorators-as-logger-and-debugging","text":"A simple way to remember the power of decorators is that the decorator (the nested function illustrated below) can (1) access the passed arguments of the decorated function and (2) access the decorated function Therefore this allows us to modify the decorated function without changing the decorated function # Create a nested function that will be our decorator def function_inspector ( func ): def inner ( * args , ** kwargs ): result = func ( * args , ** kwargs ) print ( f 'Function args: { args } ' ) print ( f 'Function kwargs: { kwargs } ' ) print ( f 'Function return result: { result } ' ) return result return inner # Decorate our multiply function with our logger for easy logging # Of arguments pass to the function and results returned @function_inspector def multiply_func ( num_one , num_two ): return num_one * num_two multiply_result = multiply_func ( num_one = 1 , num_two = 2 ) Function args: () Function kwargs: {'num_one': 1, 'num_two': 2} Function return result: 2","title":"Decorators as Logger and Debugging"},{"location":"programming/python/python/#dates","text":"","title":"Dates"},{"location":"programming/python/python/#get-current-date","text":"import datetime now = datetime . datetime . now () print ( now ) 2019-08-12 14:20:45.604849","title":"Get Current Date"},{"location":"programming/python/python/#get-clean-string-current-date","text":"# YYYY-MM-DD now . date () . strftime ( '20%y-%m- %d ' ) '2019-08-12'","title":"Get Clean String Current Date"},{"location":"programming/python/python/#count-business-days","text":"# Number of business days in a month from Jan 2019 to Feb 2019 import numpy as np days = np . busday_count ( '2019-01' , '2019-02' ) print ( days ) 23","title":"Count Business Days"},{"location":"programming/python/python/#progress-bars","text":"","title":"Progress Bars"},{"location":"programming/python/python/#tqdm","text":"Simple progress bar via pip install tqdm from tqdm import tqdm import time for i in tqdm ( range ( 100 )): time . sleep ( 0.1 ) pass 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 100/100 [00:10<00:00, 9.91it/s]","title":"TQDM"},{"location":"programming/python/python/#check-paths","text":"","title":"Check Paths"},{"location":"programming/python/python/#check-path-exists","text":"Check if directory exists import os directory = 'new_dir' print ( os . path . exists ( directory )) # Magic function to list all folders ! ls - d */ False ls: cannot access '*/': No such file or directory","title":"Check Path Exists"},{"location":"programming/python/python/#check-path-exists-otherwise-create-folder","text":"Check if directory exists, otherwise make folder if not os . path . exists ( directory ): os . makedirs ( directory ) # Magic function to list all folders ! ls - d */ # Remove directory ! rmdir new_dir new_dir/","title":"Check Path Exists Otherwise Create Folder"},{"location":"programming/python/python/#exception-handling","text":"","title":"Exception Handling"},{"location":"programming/python/python/#try-except-finally-error","text":"This is very handy and often exploited to patch up (save) poorly written code You can use general exceptions or specific ones like ValueError , KeyboardInterrupt and MemoryError to name a few value_one = 'a' value_two = 2 # Try the following line of code try : final_sum = value_one / value_two print ( 'Code passed!' ) # If the code above fails, code nested under except will be executed except : print ( 'Code failed!' ) # This will run no matter whether the nested code in try or except is executed finally : print ( 'Ran code block regardless of error or not.' ) Code failed! Ran code block regardless of error or not.","title":"Try, Except, Finally: Error"},{"location":"programming/python/python/#try-except-finally-no-error","text":"There won't be errors because you can divide 4 with 2 value_one = 4 value_two = 2 # Try the following line of code try : final_sum = value_one / value_two print ( 'Code passed!' ) # If the code above fails, code nested under except will be executed except : print ( 'Code failed!' ) # This will run no matter whether the nested code in try or except is executed finally : print ( 'Ran code block regardless of error or not.' ) Code passed! Ran code block regardless of error or not.","title":"Try, Except, Finally: No Error"},{"location":"programming/python/python/#assertion","text":"This comes in handy when you want to enforce strict requirmenets of a certain value, shape, value type, or others for i in range ( 10 ): assert i <= 5 , 'Value is more than 5, rejected' print ( f 'Passed assertion for value { i } ' ) Passed assertion for value 0 Passed assertion for value 1 Passed assertion for value 2 Passed assertion for value 3 Passed assertion for value 4 Passed assertion for value 5 --------------------------------------------------------------------------- AssertionError Traceback (most recent call last) <ipython-input-2-d9d077e139a9> in <module> 1 for i in range(10): ----> 2 assert i <= 5, 'Value is more than 5, rejected' 3 print(f'Passed assertion for value {i}') AssertionError: Value is more than 5, rejected","title":"Assertion"},{"location":"programming/python/python/#asynchronous","text":"","title":"Asynchronous"},{"location":"programming/python/python/#concurrency-parallelism-asynchronous","text":"Concurrency (single CPU core): multiple threads on a single core running in sequence , only 1 thread is making progress at any point Think of 1 human, packing a box then wrapping the box Parallelism (mutliple GPU cores): multiple threads on multiple cores running in parallel , multiple threads can be making progress Think of 2 humans, one packing a box, another wrapping the box Asynchronous: concurrency but with a more dynamic system that moves amongst threads more efficiently rather than waiting for a task to finish then moving to the next task Python's asyncio allows us to code asynchronously Benefits: Scales better if you need to wait on a lot of processes Less memory (easier in this sense) to wait on thousands of co-routines than running on thousands of threads Good for IO bound uses like reading/saving from databases while subsequently running other computation Easier management than multi-thread processing like in parallel programming In the sense that everything operates sequentially in the same memory space","title":"Concurrency, Parallelism, Asynchronous"},{"location":"programming/python/python/#asynchronous-key-components","text":"The three main parts are (1) coroutines and subroutines, (2) event loops, and (3) future. Co-routine and subroutines Subroutine: the usual function Coroutine: this allows us to maintain states with memory of where things stopped so we can swap amongst subroutines async declares a function as a coroutine await to call a coroutine Event loops Future","title":"Asynchronous Key Components"},{"location":"programming/python/python/#synchronous-2-function-calls","text":"import timeit def add_numbers ( num_1 , num_2 ): print ( 'Adding' ) time . sleep ( 1 ) return num_1 + num_2 def display_sum ( num_1 , num_2 ): total_sum = add_numbers ( num_1 , num_2 ) print ( f 'Total sum { total_sum } ' ) def main (): display_sum ( 2 , 2 ) display_sum ( 2 , 2 ) start = timeit . default_timer () main () end = timeit . default_timer () total_time = end - start print ( f 'Total time { total_time : .2f } s' ) Adding Total sum 4 Adding Total sum 4 Total time 2.00s","title":"Synchronous 2 Function Calls"},{"location":"programming/python/python/#parallel-2-function-calls","text":"from multiprocessing import Pool from functools import partial start = timeit . default_timer () pool = Pool () result = pool . map ( partial ( display_sum , num_2 = 2 ), [ 2 , 2 ]) end = timeit . default_timer () total_time = end - start print ( f 'Total time { total_time : .2f } s' ) Adding Adding Total sum 4 Total sum 4 Total time 1.08s","title":"Parallel 2 Function Calls"},{"location":"programming/python/python/#asynchronous-2-function-calls","text":"For this use case, it'll take half the time compared to a synchronous application and slightly faster than parallel application (although not always true for parallel except in this case) import asyncio import timeit import time async def add_numbers ( num_1 , num_2 ): print ( 'Adding' ) await asyncio . sleep ( 1 ) return num_1 + num_2 async def display_sum ( num_1 , num_2 ): total_sum = await add_numbers ( num_1 , num_2 ) print ( f 'Total sum { total_sum } ' ) async def main (): # .gather allows us to group subroutines await asyncio . gather ( display_sum ( 2 , 2 ), display_sum ( 2 , 2 )) start = timeit . default_timer () # For .ipynb, event loop already done await main () # For .py # asyncio.run(main()) end = timeit . default_timer () total_time = end - start print ( f 'Total time { total_time : .4f } s' ) Adding Adding Total sum 4 Total sum 4 Total time 1.0021s","title":"Asynchronous 2 Function Calls"},{"location":"programming/r/r/","text":"R \u00b6 Quick Start Basics \u00b6 Variable Object \u00b6 var_a <- 2 print ( var_a ) [1] 2 var_b = 3 print ( var_b ) [1] 3 Check Object Type \u00b6 # In this case, it's a numeric type class ( var_a ) 'numeric' List / Vector \u00b6 list_a <- c ( 0 , 1 , 2 ) list_a .list-inline {list-style: none; margin:0; padding: 0} .list-inline>li {display: inline-block} .list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex} 0 1 2 # It's inclusive start till end, unlike Python which leaves the last out list_b <- c ( 0 : 6 ) list_b .list-inline {list-style: none; margin:0; padding: 0} .list-inline>li {display: inline-block} .list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex} 0 1 2 3 4 5 6 # Similar to a Python dictionary list_c <- list ( john_height = 120 , doe_height = 150 ) list_c $john_height 120 $doe_height 150 # Get keys only via names() names ( list_c ) .list-inline {list-style: none; margin:0; padding: 0} .list-inline>li {display: inline-block} .list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex} 'john_height' 'doe_height' # Get values only via uname() unname ( list_c ) 120 150 Matrix \u00b6 # Create 2 vectors vect_a <- c ( 1 , 2 ) vect_b <- c ( 3 , 4 ) vect_a vect_b .list-inline {list-style: none; margin:0; padding: 0} .list-inline>li {display: inline-block} .list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex} 1 2 .list-inline {list-style: none; margin:0; padding: 0} .list-inline>li {display: inline-block} .list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex} 3 4 # Bind as row mat_c <- rbind ( vect_a , vect_b ) mat_c A matrix: 2 \u00d7 2 of type dbl vect_a 1 2 vect_b 3 4 # Bind as column mat_d <- cbind ( vect_a , vect_b ) mat_d A matrix: 2 \u00d7 2 of type dbl vect_a vect_b 1 3 2 4 # Create matrix mat_e <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_e A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 # Create matrix with byrow option mat_g <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 , byrow = FALSE ) mat_g A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 # Create same matrix with byrow option mat_f <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 , byrow = TRUE ) mat_f A matrix: 2 \u00d7 3 of type dbl 1 2 3 4 5 6 # Index matrix of first row and first column mat_f [ 1 , 1 ] # Index matrix of second row and first column mat_f [ 2 , 1 ] # Index matrix of second row and third column mat_f [ 2 , 3 ] # Select full first row mat_f [ 1 ,] # Select full first column mat_f [, 1 ] 1 4 6 .list-inline {list-style: none; margin:0; padding: 0} .list-inline>li {display: inline-block} .list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex} 1 2 3 .list-inline {list-style: none; margin:0; padding: 0} .list-inline>li {display: inline-block} .list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex} 1 4 # Find transpose of matrix: rows to cols t ( mat_f ) A matrix: 3 \u00d7 2 of type dbl 1 4 2 5 3 6 # Add matrix mat_1 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_2 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_new <- mat_1 + mat_2 mat_1 mat_2 mat_new A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 A matrix: 2 \u00d7 3 of type dbl 2 6 10 4 8 12 # Subtract matrix mat_1 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_2 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_new <- mat_1 - mat_2 mat_1 mat_2 mat_new A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 A matrix: 2 \u00d7 3 of type dbl 0 0 0 0 0 0 # Element-wise multiplication / Hadamard product mat_1 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_2 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_new <- mat_1 * mat_2 mat_1 mat_2 mat_new A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 A matrix: 2 \u00d7 3 of type dbl 1 9 25 4 16 36 # Matrix multiplication mat_1 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_2 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 3 , ncol = 2 ) mat_new <- mat_1 %*% mat_2 mat_1 mat_2 mat_new A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 A matrix: 3 \u00d7 2 of type dbl 1 4 2 5 3 6 A matrix: 2 \u00d7 2 of type dbl 22 49 28 64 # Element-wise division mat_1 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_2 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_new <- mat_1 / mat_2 mat_1 mat_2 mat_new A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 A matrix: 2 \u00d7 3 of type dbl 1 1 1 1 1 1 # Scalar multiplication with matrix mat_1 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_new <- 10 * mat_1 mat_1 mat_new A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 A matrix: 2 \u00d7 3 of type dbl 10 30 50 20 40 60 # Broadcast vector of 1x3 to matrix of 2x3 vec_1 <- c ( 1 , 2 , 3 ) mat_1 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) vec_1 + mat_1 A matrix: 2 \u00d7 3 of type dbl 2 6 7 4 5 9 # Check for determinant of matrix mat_A <- matrix ( c ( 1 , 2 , 3 , 4 ), nrow = 2 , ncol = 2 ) mat_A # Here det != 0, hence inverse exist det_of_mat_A = det ( mat_A ) cat ( det_of_mat_A , 'is non-zero hence the inverse exist' ) A matrix: 2 \u00d7 2 of type dbl 1 3 2 4 -2 is non-zero hence the inverse exist # Invert of Matrix: mat' mat_A_inv = solve ( mat_A ) mat_A_inv A matrix: 2 \u00d7 2 of type dbl -2 1.5 1 -0.5 # I = A'A = AA' mat_A_identity = mat_A_inv %*% mat_A mat_A_identity mat_A_inv %*% mat_A mat_A %*% mat_A_inv A matrix: 2 \u00d7 2 of type dbl 1 0 0 1 A matrix: 2 \u00d7 2 of type dbl 1 0 0 1 A matrix: 2 \u00d7 2 of type dbl 1 0 0 1 # IA = AI = A mat_A mat_A_identity %*% mat_A mat_A %*% mat_A_identity A matrix: 2 \u00d7 2 of type dbl 1 3 2 4 A matrix: 2 \u00d7 2 of type dbl 1 3 2 4 A matrix: 2 \u00d7 2 of type dbl 1 3 2 4 DataFrames \u00b6 price_var <- c ( 100 , 105 , 120 ) date_var <- as.Date ( c ( '2021-01-11' , '2021-01-12' , '2021-01-13' )) df <- data.frame ( price_var , date_var ) df A data.frame: 3 \u00d7 2 price_var date_var <dbl> <date> 100 2021-01-11 105 2021-01-12 120 2021-01-13 For Loop \u00b6 for ( i in c ( 0 : 3 )) { print ( i ) } [1] 0 [1] 1 [1] 2 [1] 3 While Loop \u00b6 # Be careful, you might end up in an infinite loop # if say i = -1 i <- 0 while ( i < 3 ) { stdout_str = paste ( 'Yet to reach 3, i is: ' , i ) print ( stdout_str ) i <- i + 1 } print ( 'Reached 3!' ) [1] \"Yet to reach 3, i is: 0\" [1] \"Yet to reach 3, i is: 1\" [1] \"Yet to reach 3, i is: 2\" [1] \"Reached 3!\" Math \u00b6 # Multiplication 2 * 2 4 # Division 2 / 2 1 # Addition 2 + 2 4 # Subtraction 2 - 2 0 # Exponentiation exp ( 10 ) 22026.4657948067 # Inbuilt value pi 2 * pi 6.28318530717959 # Natural log: ln log ( 2 ) 0.693147180559945 # Log with base 2 var_a = log ( 5 , base = 2 ) var_a 2.32192809488736 # Power to go back to 2 2 ** var_a 5 # Power 2 ** 2 4 # Square root sqrt ( 4 ) # Alternative square root via power yiled same results 4 ** 0.5 2 2 # We can square root a complex number sqrt ( -25+0i ) # But we cannot square root a real number cat ( sprintf ( '---------------' )) cat ( sprintf ( 'This will fail!' )) cat ( sprintf ( '---------------' )) sqrt ( -25 ) 0+5i ---------------This will fail!--------------- Warning message in sqrt(-25): \u201cNaNs produced\u201d NaN # Print as fractions rather than decimal library ( MASS ) fractions ( 1 - 0.25 ) \u00be Statistics \u00b6 # Create list list_a <- c ( 1 , 2 , 3 , 4 , 5 , 6 ) # Get standard deviation cat ( 'standard deviation:' , sd ( list_a )) # Get variance cat ( '\\nvariance:' , var ( list_a )) # Get mean cat ( '\\nmedian:' , mean ( list_a )) # Get median cat ( '\\nmean:' , median ( list_a )) # Get summary statistics summary ( list_a ) standard deviation: 1.870829 variance: 3.5 median: 3.5 mean: 3.5 Min. 1st Qu. Median Mean 3rd Qu. Max. 1.00 2.25 3.50 3.50 4.75 6.00 # t-test var_test = t.test ( list_a ) var_test One Sample t-test data: list_a t = 4.5826, df = 5, p-value = 0.005934 alternative hypothesis: true mean is not equal to 0 95 percent confidence interval: 1.536686 5.463314 sample estimates: mean of x 3.5 # This shows what you can call if we assign it to an object names ( t.test ( list_a )) .list-inline {list-style: none; margin:0; padding: 0} .list-inline>li {display: inline-block} .list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex} 'statistic' 'parameter' 'p.value' 'conf.int' 'estimate' 'null.value' 'stderr' 'alternative' 'method' 'data.name' # Get confidence interval var_test $ conf.int .list-inline {list-style: none; margin:0; padding: 0} .list-inline>li {display: inline-block} .list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex} 1.53668569301968 5.46331430698032 # Mid-point of confidence interval is the mean mean ( var_test $ conf.int ) 3.5 # Get p-value var_test $ p.value 0.00593354451759226 # Plot histogram, hist() doesn't work all the time barplot ( table ( list_a )) # Plot dot plot plot ( list_a ) # Plot dot plot # type: p for point, l for line, b for both # col: cyan, blue, green, red plot ( list_a , xlab = 'variable x' , ylab = 'variable y' , main = 'X-Y Plot' , type = 'b' , col = 'blue' )","title":"R"},{"location":"programming/r/r/#r","text":"","title":"R"},{"location":"programming/r/r/#quick-start-basics","text":"","title":"Quick Start Basics"},{"location":"programming/r/r/#variable-object","text":"var_a <- 2 print ( var_a ) [1] 2 var_b = 3 print ( var_b ) [1] 3","title":"Variable Object"},{"location":"programming/r/r/#check-object-type","text":"# In this case, it's a numeric type class ( var_a ) 'numeric'","title":"Check Object Type"},{"location":"programming/r/r/#list-vector","text":"list_a <- c ( 0 , 1 , 2 ) list_a .list-inline {list-style: none; margin:0; padding: 0} .list-inline>li {display: inline-block} .list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex} 0 1 2 # It's inclusive start till end, unlike Python which leaves the last out list_b <- c ( 0 : 6 ) list_b .list-inline {list-style: none; margin:0; padding: 0} .list-inline>li {display: inline-block} .list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex} 0 1 2 3 4 5 6 # Similar to a Python dictionary list_c <- list ( john_height = 120 , doe_height = 150 ) list_c $john_height 120 $doe_height 150 # Get keys only via names() names ( list_c ) .list-inline {list-style: none; margin:0; padding: 0} .list-inline>li {display: inline-block} .list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex} 'john_height' 'doe_height' # Get values only via uname() unname ( list_c ) 120 150","title":"List / Vector"},{"location":"programming/r/r/#matrix","text":"# Create 2 vectors vect_a <- c ( 1 , 2 ) vect_b <- c ( 3 , 4 ) vect_a vect_b .list-inline {list-style: none; margin:0; padding: 0} .list-inline>li {display: inline-block} .list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex} 1 2 .list-inline {list-style: none; margin:0; padding: 0} .list-inline>li {display: inline-block} .list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex} 3 4 # Bind as row mat_c <- rbind ( vect_a , vect_b ) mat_c A matrix: 2 \u00d7 2 of type dbl vect_a 1 2 vect_b 3 4 # Bind as column mat_d <- cbind ( vect_a , vect_b ) mat_d A matrix: 2 \u00d7 2 of type dbl vect_a vect_b 1 3 2 4 # Create matrix mat_e <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_e A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 # Create matrix with byrow option mat_g <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 , byrow = FALSE ) mat_g A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 # Create same matrix with byrow option mat_f <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 , byrow = TRUE ) mat_f A matrix: 2 \u00d7 3 of type dbl 1 2 3 4 5 6 # Index matrix of first row and first column mat_f [ 1 , 1 ] # Index matrix of second row and first column mat_f [ 2 , 1 ] # Index matrix of second row and third column mat_f [ 2 , 3 ] # Select full first row mat_f [ 1 ,] # Select full first column mat_f [, 1 ] 1 4 6 .list-inline {list-style: none; margin:0; padding: 0} .list-inline>li {display: inline-block} .list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex} 1 2 3 .list-inline {list-style: none; margin:0; padding: 0} .list-inline>li {display: inline-block} .list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex} 1 4 # Find transpose of matrix: rows to cols t ( mat_f ) A matrix: 3 \u00d7 2 of type dbl 1 4 2 5 3 6 # Add matrix mat_1 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_2 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_new <- mat_1 + mat_2 mat_1 mat_2 mat_new A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 A matrix: 2 \u00d7 3 of type dbl 2 6 10 4 8 12 # Subtract matrix mat_1 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_2 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_new <- mat_1 - mat_2 mat_1 mat_2 mat_new A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 A matrix: 2 \u00d7 3 of type dbl 0 0 0 0 0 0 # Element-wise multiplication / Hadamard product mat_1 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_2 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_new <- mat_1 * mat_2 mat_1 mat_2 mat_new A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 A matrix: 2 \u00d7 3 of type dbl 1 9 25 4 16 36 # Matrix multiplication mat_1 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_2 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 3 , ncol = 2 ) mat_new <- mat_1 %*% mat_2 mat_1 mat_2 mat_new A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 A matrix: 3 \u00d7 2 of type dbl 1 4 2 5 3 6 A matrix: 2 \u00d7 2 of type dbl 22 49 28 64 # Element-wise division mat_1 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_2 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_new <- mat_1 / mat_2 mat_1 mat_2 mat_new A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 A matrix: 2 \u00d7 3 of type dbl 1 1 1 1 1 1 # Scalar multiplication with matrix mat_1 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) mat_new <- 10 * mat_1 mat_1 mat_new A matrix: 2 \u00d7 3 of type dbl 1 3 5 2 4 6 A matrix: 2 \u00d7 3 of type dbl 10 30 50 20 40 60 # Broadcast vector of 1x3 to matrix of 2x3 vec_1 <- c ( 1 , 2 , 3 ) mat_1 <- matrix ( c ( 1 , 2 , 3 , 4 , 5 , 6 ), nrow = 2 , ncol = 3 ) vec_1 + mat_1 A matrix: 2 \u00d7 3 of type dbl 2 6 7 4 5 9 # Check for determinant of matrix mat_A <- matrix ( c ( 1 , 2 , 3 , 4 ), nrow = 2 , ncol = 2 ) mat_A # Here det != 0, hence inverse exist det_of_mat_A = det ( mat_A ) cat ( det_of_mat_A , 'is non-zero hence the inverse exist' ) A matrix: 2 \u00d7 2 of type dbl 1 3 2 4 -2 is non-zero hence the inverse exist # Invert of Matrix: mat' mat_A_inv = solve ( mat_A ) mat_A_inv A matrix: 2 \u00d7 2 of type dbl -2 1.5 1 -0.5 # I = A'A = AA' mat_A_identity = mat_A_inv %*% mat_A mat_A_identity mat_A_inv %*% mat_A mat_A %*% mat_A_inv A matrix: 2 \u00d7 2 of type dbl 1 0 0 1 A matrix: 2 \u00d7 2 of type dbl 1 0 0 1 A matrix: 2 \u00d7 2 of type dbl 1 0 0 1 # IA = AI = A mat_A mat_A_identity %*% mat_A mat_A %*% mat_A_identity A matrix: 2 \u00d7 2 of type dbl 1 3 2 4 A matrix: 2 \u00d7 2 of type dbl 1 3 2 4 A matrix: 2 \u00d7 2 of type dbl 1 3 2 4","title":"Matrix"},{"location":"programming/r/r/#dataframes","text":"price_var <- c ( 100 , 105 , 120 ) date_var <- as.Date ( c ( '2021-01-11' , '2021-01-12' , '2021-01-13' )) df <- data.frame ( price_var , date_var ) df A data.frame: 3 \u00d7 2 price_var date_var <dbl> <date> 100 2021-01-11 105 2021-01-12 120 2021-01-13","title":"DataFrames"},{"location":"programming/r/r/#for-loop","text":"for ( i in c ( 0 : 3 )) { print ( i ) } [1] 0 [1] 1 [1] 2 [1] 3","title":"For Loop"},{"location":"programming/r/r/#while-loop","text":"# Be careful, you might end up in an infinite loop # if say i = -1 i <- 0 while ( i < 3 ) { stdout_str = paste ( 'Yet to reach 3, i is: ' , i ) print ( stdout_str ) i <- i + 1 } print ( 'Reached 3!' ) [1] \"Yet to reach 3, i is: 0\" [1] \"Yet to reach 3, i is: 1\" [1] \"Yet to reach 3, i is: 2\" [1] \"Reached 3!\"","title":"While Loop"},{"location":"programming/r/r/#math","text":"# Multiplication 2 * 2 4 # Division 2 / 2 1 # Addition 2 + 2 4 # Subtraction 2 - 2 0 # Exponentiation exp ( 10 ) 22026.4657948067 # Inbuilt value pi 2 * pi 6.28318530717959 # Natural log: ln log ( 2 ) 0.693147180559945 # Log with base 2 var_a = log ( 5 , base = 2 ) var_a 2.32192809488736 # Power to go back to 2 2 ** var_a 5 # Power 2 ** 2 4 # Square root sqrt ( 4 ) # Alternative square root via power yiled same results 4 ** 0.5 2 2 # We can square root a complex number sqrt ( -25+0i ) # But we cannot square root a real number cat ( sprintf ( '---------------' )) cat ( sprintf ( 'This will fail!' )) cat ( sprintf ( '---------------' )) sqrt ( -25 ) 0+5i ---------------This will fail!--------------- Warning message in sqrt(-25): \u201cNaNs produced\u201d NaN # Print as fractions rather than decimal library ( MASS ) fractions ( 1 - 0.25 ) \u00be","title":"Math"},{"location":"programming/r/r/#statistics","text":"# Create list list_a <- c ( 1 , 2 , 3 , 4 , 5 , 6 ) # Get standard deviation cat ( 'standard deviation:' , sd ( list_a )) # Get variance cat ( '\\nvariance:' , var ( list_a )) # Get mean cat ( '\\nmedian:' , mean ( list_a )) # Get median cat ( '\\nmean:' , median ( list_a )) # Get summary statistics summary ( list_a ) standard deviation: 1.870829 variance: 3.5 median: 3.5 mean: 3.5 Min. 1st Qu. Median Mean 3rd Qu. Max. 1.00 2.25 3.50 3.50 4.75 6.00 # t-test var_test = t.test ( list_a ) var_test One Sample t-test data: list_a t = 4.5826, df = 5, p-value = 0.005934 alternative hypothesis: true mean is not equal to 0 95 percent confidence interval: 1.536686 5.463314 sample estimates: mean of x 3.5 # This shows what you can call if we assign it to an object names ( t.test ( list_a )) .list-inline {list-style: none; margin:0; padding: 0} .list-inline>li {display: inline-block} .list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex} 'statistic' 'parameter' 'p.value' 'conf.int' 'estimate' 'null.value' 'stderr' 'alternative' 'method' 'data.name' # Get confidence interval var_test $ conf.int .list-inline {list-style: none; margin:0; padding: 0} .list-inline>li {display: inline-block} .list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex} 1.53668569301968 5.46331430698032 # Mid-point of confidence interval is the mean mean ( var_test $ conf.int ) 3.5 # Get p-value var_test $ p.value 0.00593354451759226 # Plot histogram, hist() doesn't work all the time barplot ( table ( list_a )) # Plot dot plot plot ( list_a ) # Plot dot plot # type: p for point, l for line, b for both # col: cyan, blue, green, red plot ( list_a , xlab = 'variable x' , ylab = 'variable y' , main = 'X-Y Plot' , type = 'b' , col = 'blue' )","title":"Statistics"},{"location":"programming/sympy/calculus_sympy/","text":"Calculus wth Sympy \u00b6 In this section, we will be covering basic calculus theory and using sympy to solve the related equations. Calculus: Differentiation (Theory) \u00b6 Basic Rules \u00b6 \\[ f(x) = x^n \\\\ f\\prime(x) = nx^{x-1} \\] \\[ f(x) = c \\\\ f\\prime(x) = 0 \\] \\[ f(x) = \\cos(x) \\\\ f\\prime(x) = -\\sin(x) \\] \\[ f(x) = \\cosh(x) \\\\ f\\prime(x) = \\sinh(x) \\] \\[ f(x) = \\sin(x) \\\\ f\\prime(x) = \\cos(x) \\] \\[ f(x) = \\sinh(x) \\\\ f\\prime(x) = \\cosh(x) \\] \\[ f(x) = \\ln (x) \\\\ f\\prime(x) = \\frac{1}{x} \\] Sum Difference Rule \u00b6 Given \\[ f(x) = g(x) \\pm h(x) \\] Then \\[ f\\prime(x) = g\\prime(x) \\pm h\\prime(x) \\] Product Rule \u00b6 Given a function with two parts \\[ f(x) = g(x) \\cdot h(x) \\] First derivative \\[ f\\prime(x) = g\\prime(x) \\cdot h(x) + g(x) \\cdot h\\prime(x) \\] Given three parts, we always use the product rule to keep it to 2 parts \\[ f(x) = g(x) \\cdot h(x) \\cdot i(x) \\] \\[ f'(x) = \\big[g(x) \\cdot h(x)\\big]\\prime \\cdot i(x) + \\big[g(x) \\cdot h(x) \\big] \\cdot i\\prime(x) \\] Quotient Rule \u00b6 Given \\[ f(x) = \\frac{g(x)}{h(x)} \\] Then \\[ f\\prime(x) = \\frac{g\\prime(x) \\cdot h(x) - g(x) \\cdot h\\prime(x)}{h^2(x)} \\] Logarithmic Rule \u00b6 Given \\[f(x) = ln(g(x))\\] Then \\[ f\\prime(x) = \\frac{g\\prime(x)}{g(x)} \\] Exponential Rule \u00b6 Given \\[ f(x) = \\exp \\big(g(x)\\big) \\] Then \\[ f\\prime(x) = g\\prime(x) \\exp \\big(g(x)\\big) \\] Partial Derivative \u00b6 Given \\[ f(x) = g(x_{i}, x_{i+1}, \\dots, x_n) \\] When computing the partial derivative of \\(x_i\\) with respect to (w.r.t.) the multivariable function \\(f(x_i \\dots x_n)\\) , treat the rest of the variables as constant. Calculus: Integration (Theory) \u00b6 Basic Rules \u00b6 Given function \\(f(x)\\) and integral of that function \\(F(x)\\) , then basic rules include \\[ f(x) = x^n \\\\ F(x) = \\frac{x^{n+1}}{n+1} + c \\] \\[ f(x) = 0 \\\\ F(x) = c \\] \\[ f(x) = e^x \\\\ F(x) = e^x + c \\] \\[ f(x) = a^x \\\\ F(x) = \\frac{a^x}{\\ln a} + c \\] \\[ f(x) = \\sin(x) \\\\ F(x) = -\\cos(x) + c \\] \\[ f(x) = \\sinh(x) \\\\ F(x) = \\cosh(x) + c \\] \\[ f(x) = \\cos(x) \\\\ F(x) = \\sin(x) + c \\] \\[ f(x) = \\cosh(x) \\\\ F(x) = \\sinh(x) + c \\] \\[ f(x) = \\frac{1}{x} \\\\ F(x) = \\ln\\left|x\\right| + c \\] \\[ f(x) = m \\\\ F(x) = mx + c \\] \\[ f(x) = \\frac{1}{x^2 + a^2} \\\\ F(x) = (\\frac{1}{a}) \\cdot \\arctan(\\frac{x}{a}) + c \\] \\[ f(x) = \\frac{1}{\\sqrt{a^2 - x^2}} \\\\ F(x) = \\arcsin(\\frac{x}{a}) + c \\] Sum Difference Rule \u00b6 Given \\[ f(x) = g(x) \\pm h(x) \\] Then \\[ F(x) = \\int \\big( g(x) \\pm h(x) \\big) \\, dx\\] Integration by Parts \u00b6 \\[ F(x) = \\int u\\ v\\prime \\ dx = u\\ v - \\int v\\ u\\prime \\ dx \\] Definite Integral Rules \u00b6 Zero integral when not moving integration point \u00b6 \\[ \\int _a ^a f(x) \\ dx = 0 \\] Definite integral's interval switching becomes negative \u00b6 \\[ \\int _a ^b f(x) \\ dx = - \\int _b ^a f(x) \\ dx \\] Definite integral decomposed into parts \u00b6 \\[ \\int _a ^b f(x) \\ dx = \\int _a ^c f(x) \\ dx - \\int _b ^c f(x) \\ dx \\] Substitution method for solving definite integrals \u00b6 Substitution Example 1 \u00b6 Given $$ f(x) = x \\cos(x^2 + 1) $$ Then definite integral from -1 to 1 is $$ F(x)_{-1}^1 = \\int _{-1} ^ 1 x \\cos(x^2 + 1) dx $$ Modifying the equation to make \\(du\\) and \\(u\\) substitution method results in $$ F(x)_{-1}^1 = \\frac{1}{2} \\int _{-1} ^ 1 2x \\cos(x^2 + 1) dx $$ Let \\(u\\) be $$ u = x^2 +1 $$ Let \\(du\\) be $$ du = 2x $$ New limits given -1 and 1 $$ u = (-1)^2 + 1 = 2 $$ $$ u = (1)^2 + 1 = 2 $$ Then by zero integral rule $$ F(x)_{-1}^1 = \\int _2 ^2 cos(u) du = 0 $$ Substitution Example 2 \u00b6 Given $$ h(x) = x(x+3)^{\\frac{1}{2}} $$ Then definite integral from -1 to 1 is $$ H(x)_{-1}^1 = \\int _{-1} ^ 1 x(x+3)^{\\frac{1}{2}} $$ Let \\(u\\) be $$ u = x + 3 $$ $$ x = u - 3 $$ Let \\(du\\) be $$ du = 1 $$ New limits given -1 and 1 $$ u = -1 + 3 = 2 $$ $$ u = 1 + 3 = 4 $$ Then \\[ H(x)_{-1}^1 = \\int _{2} ^ 4 (u-3)(u)^{\\frac{1}{2}} \\ du \\] \\[ H(x)_{-1}^1 = \\int _{2} ^ 4 u^{\\frac{3}{2}} - 3u^{\\frac{1}{2}} \\ du \\] \\[ H(x)_{-1}^1 = \\big[ \\frac{u^{\\frac{5}{2}}}{\\frac{5}{2}} - \\frac{3u^{\\frac{3}{2}}}{\\frac{3}{2}} \\big]_2 ^4 \\] \\[ H(x)_{-1}^1 = \\frac{2}{5} [-8 + 6\\sqrt(2)] \\] Substitution Example 3 \u00b6 Keep on substituting! You can keep running the substitution rule multiple times, where you can go from \\(u\\) to \\(z\\) to etc. to solve the definite integral. It is important to take note that your limits change every time you run the substitution rule once! Multiple Integrals \u00b6 This is used for functions with multiple variables. Hence, instead of the usual single variable where the definite integral represents the area under the curve, multiple integrals calculate hypervolumes with multiple dimensional functions. The key is in integrating step by step, when integrating with respect to each variable, the rest of the variables act as constants. Double Integral \u00b6 \\[V = \\int _{x=a}^{x=b} \\int _{y=c}^{y=d} f(x,y) \\ dy \\ dx\\] Triple Integral \u00b6 \\[V = \\int _{x=a}^{x=b} \\int _{y=c}^{y=d} \\int _{z=e}^{z=f} f(x,y,z) \\ dz \\ dy \\ dx\\] Calculus: Taylor Expansion (Theory) \u00b6 Taylor Expansion: Single Variable \u00b6 Given a function that has continuous derivatives up to \\((n+1)\\) order, the function can be expanded with the following equation: \\[ F(x) = \\sum^{\\infty}_{n=0} \\frac{F^{(n)}(a)}{n!}\\ (x-a)^n \\] \\[ F(x) = F(a) + F\\prime (x)(x-a) + \\frac{F\\prime\\prime(a)}{2!}(x-a)^2 + \\cdots + \\frac{F(n)(a)}{n!} + R_n(x) \\] If not expanded till \\(\\infty\\) then there will be a remainder of \\(R_n(x)\\) Taylor Expansion: Multiple Variables \u00b6 In the case of 2 variables \\((x, y)\\) , we can expand the multivariate equation with the generalized Taylor expansion equation: \\[ F(x, y) = \\\\ F(a, b) + \\big[ \\frac{\\partial}{\\partial x} F(x, y)_{a,b} (x-a) + \\frac{\\partial}{\\partial y} F(x, y)_{a,b} ) (y - b) \\big] \\\\ + \\frac{1}{2!} \\big[ \\frac{\\partial ^2}{\\partial x^2} F(x, y)_{a,b} (x-a)^2 + \\frac{\\partial ^2}{\\partial y^2} F(x, y)_{a,b}(y-b)^2 \\big] \\\\ + \\frac{\\partial ^2}{\\partial x \\partial y} F(x, y)_{a,b} (x-a)(y-b) + R_2 (x,y) \\] Maclaurin Expansion \u00b6 It is simply a Taylor expansion about the point 0.","title":"Sympy"},{"location":"programming/sympy/calculus_sympy/#calculus-wth-sympy","text":"In this section, we will be covering basic calculus theory and using sympy to solve the related equations.","title":"Calculus wth Sympy"},{"location":"programming/sympy/calculus_sympy/#calculus-differentiation-theory","text":"","title":"Calculus: Differentiation (Theory)"},{"location":"programming/sympy/calculus_sympy/#basic-rules","text":"\\[ f(x) = x^n \\\\ f\\prime(x) = nx^{x-1} \\] \\[ f(x) = c \\\\ f\\prime(x) = 0 \\] \\[ f(x) = \\cos(x) \\\\ f\\prime(x) = -\\sin(x) \\] \\[ f(x) = \\cosh(x) \\\\ f\\prime(x) = \\sinh(x) \\] \\[ f(x) = \\sin(x) \\\\ f\\prime(x) = \\cos(x) \\] \\[ f(x) = \\sinh(x) \\\\ f\\prime(x) = \\cosh(x) \\] \\[ f(x) = \\ln (x) \\\\ f\\prime(x) = \\frac{1}{x} \\]","title":"Basic Rules"},{"location":"programming/sympy/calculus_sympy/#sum-difference-rule","text":"Given \\[ f(x) = g(x) \\pm h(x) \\] Then \\[ f\\prime(x) = g\\prime(x) \\pm h\\prime(x) \\]","title":"Sum Difference Rule"},{"location":"programming/sympy/calculus_sympy/#product-rule","text":"Given a function with two parts \\[ f(x) = g(x) \\cdot h(x) \\] First derivative \\[ f\\prime(x) = g\\prime(x) \\cdot h(x) + g(x) \\cdot h\\prime(x) \\] Given three parts, we always use the product rule to keep it to 2 parts \\[ f(x) = g(x) \\cdot h(x) \\cdot i(x) \\] \\[ f'(x) = \\big[g(x) \\cdot h(x)\\big]\\prime \\cdot i(x) + \\big[g(x) \\cdot h(x) \\big] \\cdot i\\prime(x) \\]","title":"Product Rule"},{"location":"programming/sympy/calculus_sympy/#quotient-rule","text":"Given \\[ f(x) = \\frac{g(x)}{h(x)} \\] Then \\[ f\\prime(x) = \\frac{g\\prime(x) \\cdot h(x) - g(x) \\cdot h\\prime(x)}{h^2(x)} \\]","title":"Quotient Rule"},{"location":"programming/sympy/calculus_sympy/#logarithmic-rule","text":"Given \\[f(x) = ln(g(x))\\] Then \\[ f\\prime(x) = \\frac{g\\prime(x)}{g(x)} \\]","title":"Logarithmic Rule"},{"location":"programming/sympy/calculus_sympy/#exponential-rule","text":"Given \\[ f(x) = \\exp \\big(g(x)\\big) \\] Then \\[ f\\prime(x) = g\\prime(x) \\exp \\big(g(x)\\big) \\]","title":"Exponential Rule"},{"location":"programming/sympy/calculus_sympy/#partial-derivative","text":"Given \\[ f(x) = g(x_{i}, x_{i+1}, \\dots, x_n) \\] When computing the partial derivative of \\(x_i\\) with respect to (w.r.t.) the multivariable function \\(f(x_i \\dots x_n)\\) , treat the rest of the variables as constant.","title":"Partial Derivative"},{"location":"programming/sympy/calculus_sympy/#calculus-integration-theory","text":"","title":"Calculus: Integration (Theory)"},{"location":"programming/sympy/calculus_sympy/#basic-rules_1","text":"Given function \\(f(x)\\) and integral of that function \\(F(x)\\) , then basic rules include \\[ f(x) = x^n \\\\ F(x) = \\frac{x^{n+1}}{n+1} + c \\] \\[ f(x) = 0 \\\\ F(x) = c \\] \\[ f(x) = e^x \\\\ F(x) = e^x + c \\] \\[ f(x) = a^x \\\\ F(x) = \\frac{a^x}{\\ln a} + c \\] \\[ f(x) = \\sin(x) \\\\ F(x) = -\\cos(x) + c \\] \\[ f(x) = \\sinh(x) \\\\ F(x) = \\cosh(x) + c \\] \\[ f(x) = \\cos(x) \\\\ F(x) = \\sin(x) + c \\] \\[ f(x) = \\cosh(x) \\\\ F(x) = \\sinh(x) + c \\] \\[ f(x) = \\frac{1}{x} \\\\ F(x) = \\ln\\left|x\\right| + c \\] \\[ f(x) = m \\\\ F(x) = mx + c \\] \\[ f(x) = \\frac{1}{x^2 + a^2} \\\\ F(x) = (\\frac{1}{a}) \\cdot \\arctan(\\frac{x}{a}) + c \\] \\[ f(x) = \\frac{1}{\\sqrt{a^2 - x^2}} \\\\ F(x) = \\arcsin(\\frac{x}{a}) + c \\]","title":"Basic Rules"},{"location":"programming/sympy/calculus_sympy/#sum-difference-rule_1","text":"Given \\[ f(x) = g(x) \\pm h(x) \\] Then \\[ F(x) = \\int \\big( g(x) \\pm h(x) \\big) \\, dx\\]","title":"Sum Difference Rule"},{"location":"programming/sympy/calculus_sympy/#integration-by-parts","text":"\\[ F(x) = \\int u\\ v\\prime \\ dx = u\\ v - \\int v\\ u\\prime \\ dx \\]","title":"Integration by Parts"},{"location":"programming/sympy/calculus_sympy/#definite-integral-rules","text":"","title":"Definite Integral Rules"},{"location":"programming/sympy/calculus_sympy/#zero-integral-when-not-moving-integration-point","text":"\\[ \\int _a ^a f(x) \\ dx = 0 \\]","title":"Zero integral when not moving integration point"},{"location":"programming/sympy/calculus_sympy/#definite-integrals-interval-switching-becomes-negative","text":"\\[ \\int _a ^b f(x) \\ dx = - \\int _b ^a f(x) \\ dx \\]","title":"Definite integral's interval switching becomes negative"},{"location":"programming/sympy/calculus_sympy/#definite-integral-decomposed-into-parts","text":"\\[ \\int _a ^b f(x) \\ dx = \\int _a ^c f(x) \\ dx - \\int _b ^c f(x) \\ dx \\]","title":"Definite integral decomposed into parts"},{"location":"programming/sympy/calculus_sympy/#substitution-method-for-solving-definite-integrals","text":"","title":"Substitution method for solving definite integrals"},{"location":"programming/sympy/calculus_sympy/#substitution-example-1","text":"Given $$ f(x) = x \\cos(x^2 + 1) $$ Then definite integral from -1 to 1 is $$ F(x)_{-1}^1 = \\int _{-1} ^ 1 x \\cos(x^2 + 1) dx $$ Modifying the equation to make \\(du\\) and \\(u\\) substitution method results in $$ F(x)_{-1}^1 = \\frac{1}{2} \\int _{-1} ^ 1 2x \\cos(x^2 + 1) dx $$ Let \\(u\\) be $$ u = x^2 +1 $$ Let \\(du\\) be $$ du = 2x $$ New limits given -1 and 1 $$ u = (-1)^2 + 1 = 2 $$ $$ u = (1)^2 + 1 = 2 $$ Then by zero integral rule $$ F(x)_{-1}^1 = \\int _2 ^2 cos(u) du = 0 $$","title":"Substitution Example 1"},{"location":"programming/sympy/calculus_sympy/#substitution-example-2","text":"Given $$ h(x) = x(x+3)^{\\frac{1}{2}} $$ Then definite integral from -1 to 1 is $$ H(x)_{-1}^1 = \\int _{-1} ^ 1 x(x+3)^{\\frac{1}{2}} $$ Let \\(u\\) be $$ u = x + 3 $$ $$ x = u - 3 $$ Let \\(du\\) be $$ du = 1 $$ New limits given -1 and 1 $$ u = -1 + 3 = 2 $$ $$ u = 1 + 3 = 4 $$ Then \\[ H(x)_{-1}^1 = \\int _{2} ^ 4 (u-3)(u)^{\\frac{1}{2}} \\ du \\] \\[ H(x)_{-1}^1 = \\int _{2} ^ 4 u^{\\frac{3}{2}} - 3u^{\\frac{1}{2}} \\ du \\] \\[ H(x)_{-1}^1 = \\big[ \\frac{u^{\\frac{5}{2}}}{\\frac{5}{2}} - \\frac{3u^{\\frac{3}{2}}}{\\frac{3}{2}} \\big]_2 ^4 \\] \\[ H(x)_{-1}^1 = \\frac{2}{5} [-8 + 6\\sqrt(2)] \\]","title":"Substitution Example 2"},{"location":"programming/sympy/calculus_sympy/#substitution-example-3","text":"Keep on substituting! You can keep running the substitution rule multiple times, where you can go from \\(u\\) to \\(z\\) to etc. to solve the definite integral. It is important to take note that your limits change every time you run the substitution rule once!","title":"Substitution Example 3"},{"location":"programming/sympy/calculus_sympy/#multiple-integrals","text":"This is used for functions with multiple variables. Hence, instead of the usual single variable where the definite integral represents the area under the curve, multiple integrals calculate hypervolumes with multiple dimensional functions. The key is in integrating step by step, when integrating with respect to each variable, the rest of the variables act as constants.","title":"Multiple Integrals"},{"location":"programming/sympy/calculus_sympy/#double-integral","text":"\\[V = \\int _{x=a}^{x=b} \\int _{y=c}^{y=d} f(x,y) \\ dy \\ dx\\]","title":"Double Integral"},{"location":"programming/sympy/calculus_sympy/#triple-integral","text":"\\[V = \\int _{x=a}^{x=b} \\int _{y=c}^{y=d} \\int _{z=e}^{z=f} f(x,y,z) \\ dz \\ dy \\ dx\\]","title":"Triple Integral"},{"location":"programming/sympy/calculus_sympy/#calculus-taylor-expansion-theory","text":"","title":"Calculus: Taylor Expansion (Theory)"},{"location":"programming/sympy/calculus_sympy/#taylor-expansion-single-variable","text":"Given a function that has continuous derivatives up to \\((n+1)\\) order, the function can be expanded with the following equation: \\[ F(x) = \\sum^{\\infty}_{n=0} \\frac{F^{(n)}(a)}{n!}\\ (x-a)^n \\] \\[ F(x) = F(a) + F\\prime (x)(x-a) + \\frac{F\\prime\\prime(a)}{2!}(x-a)^2 + \\cdots + \\frac{F(n)(a)}{n!} + R_n(x) \\] If not expanded till \\(\\infty\\) then there will be a remainder of \\(R_n(x)\\)","title":"Taylor Expansion: Single Variable"},{"location":"programming/sympy/calculus_sympy/#taylor-expansion-multiple-variables","text":"In the case of 2 variables \\((x, y)\\) , we can expand the multivariate equation with the generalized Taylor expansion equation: \\[ F(x, y) = \\\\ F(a, b) + \\big[ \\frac{\\partial}{\\partial x} F(x, y)_{a,b} (x-a) + \\frac{\\partial}{\\partial y} F(x, y)_{a,b} ) (y - b) \\big] \\\\ + \\frac{1}{2!} \\big[ \\frac{\\partial ^2}{\\partial x^2} F(x, y)_{a,b} (x-a)^2 + \\frac{\\partial ^2}{\\partial y^2} F(x, y)_{a,b}(y-b)^2 \\big] \\\\ + \\frac{\\partial ^2}{\\partial x \\partial y} F(x, y)_{a,b} (x-a)(y-b) + R_2 (x,y) \\]","title":"Taylor Expansion: Multiple Variables"},{"location":"programming/sympy/calculus_sympy/#maclaurin-expansion","text":"It is simply a Taylor expansion about the point 0.","title":"Maclaurin Expansion"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\u200b\\-_,:!=\\[\\]()\"`/]+|\\.(?!\\d)|&[lg]t;|(?!\\b)(?=[A-Z][a-z])","pipeline":["trimmer","stopWordFilter"]},"docs":[{"location":"about/","title":"About Us","text":"<p>We deploy a top-down approach that enables you to grasp deep learning and deep reinforcement learning theories and code easily and quickly. We have open-sourced all our materials through our Deep Learning Wizard Tutorials. For visual learners, feel free to sign up for our video course and join over 6000 deep learning wizards.</p> <p>To this date, we have taught thousands of students across more than 120+ countries from students in high school to postgraduates and professionals in leading MNCs and research institutions around the world.</p>"},{"location":"about/#experienced-research-and-applied-core-team","title":"Experienced Research and Applied Core Team","text":"<p>Ritchie Ng</p> <p>Industry: As the Group Head of Data &amp; AI at Eastspring Investments, I spearhead engineering, governance, and strategy across data and machine learning in 13 countries with Assets Under Management (AUM) exceeding USD 220 billion.</p> <p>Academia: Serving in the capacity of a Visiting Research Scholar at NExT++ Lab, jointly setup between NUS and Tsinghua University, I focus on applied deep learning research for Asian systematic trading strategies and the development of financial Large Language Models (LLM) using multimodal and multilingual data.</p> <p>Education: Master's degree from Imperial College London and Bachelor's degree from NUS.</p> <p>About: ritchieng.live</p> <p>Notes: ritchieng.com</p> <p>Guides: deeplearningwizard.com</p> <p>Jie Fu</p> <p>Jie Fu is a happy and funny machine learning researcher (principal investigator), at Hong Kong University of Science and Technology (HKUST), with his human-friendly big AI dream.</p> <p>He worked as a postdoc with Yoshua Bengio at University of Montreal, Quebec AI Institute (Mila), funded by Microsoft Research Montreal. He was an IVADO postdoc fellow working with Chris Pal at Polytechnique Montreal, Quebec AI Institute (Mila). He obtained his PhD from National University of Singapore under the supervision of Tat-Seng Chua. He received ICLR 2021 Outstanding Paper Award.</p> <p>He is currently working towards system-2 deep learning and its adaption to various real-world tasks, including AI for Science (drug discovery in particular) and AI for Social Good that can benefit all of society. He is also broadly interested in general deep learning, reinforcement learning, and language processing.</p> <p>Check out my profile link at bigaidream.github.io</p>"},{"location":"about/#supporters","title":"Supporters","text":"<p>Alfredo Canziani</p> <p>Alfredo is the main person who inspired Ritchie to open-source Deep Learning Wizard's materials. We are very grateful for his advice particular in the space of open-source projects, deep learning and PyTorch.</p> <p>He is currently working as an Assistant Professor at Courant Institute of Mathematical Sciences, under the guidance of professor Yann LeCun.</p> <p>Do check out his latest mini-course on PyTorch that was held in Princeton University and was delivered with Ritchie Ng at AMMI (AIMS) Kigali, Rwanda supported by Google and Facebook.</p> <p>Marek Bardonski</p> <p>Since graduation, Marek has been working on state-of-the-art research. He was involved in inventing the system that diagnoses problems with Ubuntu packages using Machine Learning and in improving the detection accuracy of breast cancer using Deep Learning, each time leading 40 highly skilled Machine Learning/Deep Learning engineers for over 6 months.</p> <p>NASA noticed his unique skills and asked him to manage a project aiming to improve the International Space Station rotation algorithm that would allow them to collect enough power from solar panels while preventing them from overheating. The project was successful and generated a huge ROI for NASA. Marek has been honored by multiple Harvard professors and the Director of Advanced Exploration Systems at NASA, who signed an official Letter of Recommendation.</p> <p>Since his first summer of university coursework, Marek has participated in four internships at NVIDIA HQ and Microsoft HQ. He also spent one year in NVIDIA Switzerland optimizing Deep Learning inference for self-driving cars and, by implementing very efficient Deep Learning algorithms in the GPU assembly language, improving the speed of Tesla\u2019s cars self-driving engine inference. He's currently the Head of AI at Sigmoidal.</p> <p>\"NVIDIA Inception Partner</p> <p></p> <p>Deep Learning Wizard is proudly an NVIDIA Inception Partner, empowering people across the world to leverage on deep learning via open-source programming languages and frameworks to solve problems.</p> <p>Amazon AWS Activate Programme Startup</p> <p></p> <p>Deep Learning Wizard is also supported by the Amazon AWS Activate Programme, Portfolio Plus, allowing us to have the infrastructure to scale and grow.   </p>"},{"location":"about/#citation","title":"Citation","text":"<p>If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.</p> <p></p>"},{"location":"consultancy/","title":"Training, Consultancy &amp; Deployment","text":"<p>We've empowered hundreds of clients from large start-ups, to MNCs, to educational institutions, and to government organizations with deep learning.</p> <p>If you require on-site training, consultancy or deployment on deep learning, deep reinforcement learning, machine learning, and/or general data science projects, please feel free to contact us at ritchie@deeplearningwizard.com.</p>"},{"location":"home/","title":"Learn The Whole AI Pipeline With","text":"<p>You will the learn the suite of tools to build an end-to-end deep learning pipeline.</p> Tools You Will Learn To Use <ul> <li>PyTorch (CPU/GPU)</li> <li>Scikit-learn (CPU)</li> <li>RAPIDS cuML (GPU)</li> <li>Gym (CPU)</li> <li>Python (scripting)</li> <li>C++ (programming)</li> <li>Bash (scripting)</li> <li>NumPy (CPU)</li> <li>CuPy (GPU)</li> <li>Pandas (CPU)</li> <li>RAPIDS cuDF (GPU)</li> <li>Matplotlib (Plot)</li> <li>Plotly (Plot)</li> <li>Streamlit (Dashboard)</li> <li>CassandraDB (CPU DB)</li> <li>BlazingSQL (GPU DB)</li> </ul> <p>We deploy a top-down approach that enables you to grasp deep learning and deep reinforcement learning theories and code easily and quickly. We have open-sourced all our materials through our Deep Learning Wizard Tutorials. For visual learners, feel free to sign up for our video course and join over 6000 deep learning wizards.</p> <p>To this date, we have taught thousands of students across more than 120+ countries from students in high school to postgraduates and professionals in leading MNCs and research institutions around the world.</p> <p></p> <ol> <li> <p>Simulation of deep reinforcement learning agent mastering games like Super Mario Bros, Flappy Bird and PacMan. These games have APIs for algorithms to interact with the environment, and they are created by talented people so feel free to check out their respective repositories with the links given.\u00a0\u21a9</p> </li> </ol>"},{"location":"pipeline/","title":"CPU to GPU Production-level Pipeline for AI","text":"<p>At Deep Learning Wizard, we cover the basics of some parts of the whole tech stack for production-level CPU/GPU-powered AI.</p> <p>This AI pipeline is entirely based on open-source distributions.</p> <p>This stack would get you started, and enable you to adjust the stack according to your needs.</p> <p></p>"},{"location":"review/","title":"Reviews","text":"<p>To this date, we have taught thousands of students across more than 120+ countries from students as young as 15 to postgraduates and professionals in leading MNCs and research institutions around the world.</p> <p>These are just some of the hundreds of reviews we've had and will be updated when we have more time or contributors.</p> <p>Roberto Trevi\u00f1o Cervantes</p> <p>Congratulations! You have achieved the unachievable, teaching a 15 year old boy how to get a machine to estimate a function, just what he had been trying to do for almost a year.</p> <p>Muktabh Mayank</p> <p>This course helped me understand idiomatic pytorch and avoiding translating theano-to-torch.</p> <p>Charles Neiswender</p> <p>I really feel like I can take the techniques I learned and apply them to deep learning projects of my choosing.</p> <p>Ian Lipton</p> <p>This was a good overview of the different types of neural networks. I like that the instructor started with linear and logistic regression before diving into the neural networks. That background helped make sense of the neural network functionality. I like that this gave enough detail on the mathematics so that you can understand the basic functioning of the models but didn't overwhelm you with the nitty gritty math.</p> <p>And check out hundreds of more reviews for our video course! </p>"},{"location":"data_engineering/nosql/cassandra/intro/","title":"Apache Cassandra","text":"Tools You Will Learn To Use <ul> <li>Apache Cassandra</li> </ul>"},{"location":"data_engineering/nosql/cassandra/intro/#reasons-for-using-apache-cassandra","title":"Reasons for Using Apache Cassandra","text":"<ul> <li>It is is an open-source NoSQL database management system (DBMS) that is designed to handle large amounts of data across many servers, providing high availability with no single point of failure. <ul> <li>Meaning if one server fails, everything goes as per normal as data is shared amongst nodes (servers).</li> <li>Compared to other master/slave models like MongoDB, Cassandra has a master-less model.</li> </ul> </li> <li>It has extremely fast read/write speeds.<ul> <li>You read/write millions of rows of data easily.</li> </ul> </li> </ul> <p>Work in progress</p> <p>This open-source portion is still a work in progress. Stay tuned while we gradually upload our tutorials and notes. Feel free to contact Ritchie Ng if you would like to contribute via our Facebook page.</p>"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/","title":"Setting up Cassandra Multi-node Cluster","text":""},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#install-apache-cassandra","title":"Install Apache Cassandra","text":"<p>Patience is key initially</p> <p>It will be quite tedious to many people in setting up a multi-node Cassandra cluster. You are likely to face a lot of problems. But with a strong community and this tried-and-proven guide, you should be ok! Just be patient.</p> <p>Remember when you're asking for help on StackOverflow, here or anywhere else, do paste the logs to make everyone's life easier to help you debug quickly. </p> <p>All you need to do is to run this base command and paste the logs.</p> <pre><code>cat /var/log/cassandra/system.log\n</code></pre> <p>Here, we will install all of the required Debian packages. To ensure you've the latest version of Cassandra, please use the official link at cassandra.apache.org/download.</p> <p>Bash Commands <pre><code>echo \"deb http://www.apache.org/dist/cassandra/debian 311x main\" | sudo tee -a /etc/apt/sources.list.d/cassandra.sources.list\n\ncurl https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add -\n\nsudo apt-get update\n\nsudo apt-get install cassandra\n</code></pre></p> <p>This would get you up and running with Cassandra v3.11.</p>"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#install-datastax-python-cassandra-driver","title":"Install DataStax Python Cassandra Driver","text":"<p>We need to install the Python Cassandra Driver so we can easily interact with our Cassandra Database without using the CQLSH (this is Cassandra's shell to run CQL commands to perform CRUD operations) commandline directly.</p> <p>What is CRUD?</p> <p>CRUD is the acronym of CREATE, READ, UPDATE, and DELETE. For databases, we typically have these 4 categories of operations so we can create new data points, update, read or delete them in our database.</p> <p>Bash Commands <pre><code>pip install cassandra-driver\n</code></pre></p> <p>This is strictly for installation on Linux platforms, refer to the official website for more details.</p>"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#single-node-cassandra-cluster","title":"Single Node Cassandra Cluster","text":"<p>Before we venture into the cool world of multi-node cluster requiring many servers, we will start with a single node cluster that only requires a single server (desktop/laptop).</p> <p>Once you've installed everything so far, you should run the following.</p> <p>Bash commands</p> <pre><code>sudo service cassandra start\nsudo nodetool status\n</code></pre> <p>And this will print out something like that:</p> <pre><code>Datacenter: datacenter1\n=============================\nStatus=Up/Down\n|/ State=Normal/Leaving/Joining/Moving\n--  Address       Load       Tokens       Owns (effective)  Host ID                               Rack\nUN  127.0.0.1     318.78 KiB  256          100.0%            g5ac4c9-99b7-65d-24cfd82524f9      rack1\n</code></pre> <p>That's it, we've built our single node Cassandra database.</p>"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#multi-node-cassandra-cluster","title":"Multi-node Cassandra Cluster","text":"<p>Remember it's standard to have at least 3 nodes, and in a basic 3 separate server configuration (3 separate desktops for non-enterprise users). Because of this, we will base this tutorial on setting up a 3-node cluster. But the same steps apply if you want to even do 10000 nodes.</p> <p>Installing Sublime critical</p> <p>Sublime is a text editor, and it would help if you're not familiar with VIM which I use frequently. Honestly using Sublime to edit all the following files we will edit is much easier, trust me! </p> <p>To install Sublime run the following bash commands in sequence: <pre><code>wget -qO - https://download.sublimetext.com/sublimehq-pub.gpg | sudo apt-key add -\n\necho \"deb https://download.sublimetext.com/ apt/stable/\" | sudo tee /etc/apt/sources.list.d/sublime-text.list\n\nsudo apt-get update\n\nsudo apt-get install sublime-text\n</code></pre></p> <p>Now check if it works by running this command in your bash: <pre><code>subl\n</code></pre></p> <p>This should open Sublime text editor! You're now ready.</p> <p>Before we move on to the steps to set up each node, we need the IP address of all 3 servers. This is simple, just run the following bash command:</p> <pre><code>ifconfig | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\\.){3}[0-9]*).*/\\2/p'\n</code></pre> <p>This would return the server's IP address that we need for example:</p> <pre><code>server_1_ip: 112.522.6.61\nserver_2_ip: 112.522.6.62\nserver_3_ip: 112.522.6.63\n</code></pre> <p>We will be using these IPs as a base for configuration in the subsequent sections. Take note yours would differ and you need to change accordingly.</p>"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#steps-per-node","title":"Steps Per Node","text":"<p>Critical Section You Need To Repeat</p> <p>This is a critcal section. On EVERY server, you need to repeat all the steps shown in this section. In our case of a 3 node cluster, using 3 servers, we need to repeat this 3 times.</p>"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#step-1-modify-cassandra-configuration-settings","title":"Step 1: Modify Cassandra Configuration Settings","text":"<p>Run the following bash to edit the configuration file for server 1 (112.522.6.61):</p> <pre><code>cd /etc/cassandra\nsubl cassandra.yaml\n</code></pre> <p>Now you need to find the following fields and change them accordingly.</p> <pre><code>cluster_name: 'CassandraDBCluster'\n\n\nseed_provider:\n  - class_name: org.apache.cassandra.locator.SimpleSeedProvider\n    parameters:\n         - seeds: \"112.522.6.61, 112.522.6.62, 112.522.6.63\"\n\n\nlisten_address: 112.522.6.61\n\n\nrpc_address: 112.522.6.61\n\n\nendpoint_snitch: GossipingPropertyFileSnitch\n\nauto_bootstrap: true\n</code></pre>"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#step-2-modify-rack-details","title":"Step 2: Modify Rack Details","text":"<p>Run the following bash command to edit the rack details:</p> <pre><code>cd /etc/cassandra\nsubl cassandra-rackdc.properties\n</code></pre> <p>And change the following fields to this: <pre><code>dc=datacenter1\n</code></pre></p>"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#step-3-check-status","title":"Step 3: Check Status","text":"<p>Restart service with the following bash commands:</p> <pre><code>sudo rm -rf /var/lib/cassandra/data/system/*\nsudo service cassandra restart\nsudo nodetool status\n</code></pre> <p>Here you would see an output like this:</p> <pre><code>Datacenter: datacenter\n=============================\nStatus=Up/Down\n|/ State=Normal/Leaving/Joining/Moving\n--  Address       Load       Tokens       Owns (effective)  Host ID                               Rack\nUN  112.522.6.61  318.78 KiB  256          100.0%            f5c84c9-99b7-45d-8856-24cfd82523f9  rack1\nUN  112.522.6.62  206.42 KiB  256          100.0%            ac2f24da-1b2c-4e3-8a75-0e28ec366a4  rack1\nUN  112.522.6.63  338.34 KiB  256          100.0%            55a227d-ffbe-45d5-8698-60c374b3a6b  rack1\n</code></pre>"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#step-4-configure-firewall-ip-settings","title":"Step 4: Configure Firewall IP Settings","text":"<p>Run the following bash commands</p> <pre><code>sudo apt-get install iptables-persistent\n\nsudo iptables -A INPUT -p tcp -s 112.522.6.62 -m multiport --dports 7000,9042 -m state --state NEW,ESTABLISHED -j ACCEPT\nsudo iptables -A INPUT -p tcp -s 112.522.6.63 -m multiport --dports 7000,9042 -m state --state NEW,ESTABLISHED -j ACCEPT\n\nsudo bash -c \"iptables-save &gt; /etc/iptables/rules.v4\"\n\nsudo netfilter-persistent reload\n\nsudo nodetool status\n</code></pre>"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#repeat-steps-1-to-4","title":"Repeat Steps 1 to 4","text":"<p>You have to repeat all 4 steps for the other 2 servers but step 1 and step 4 requires slightly different settings. For the other servers, you need to basically change the IP to the server's IP. And for the IP firewall settings, you need to allow entry from the other 2 servers instead of itself.</p>"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#server-2-example-for-step-2","title":"Server 2 Example for Step 2","text":"<pre><code>cluster_name: 'CassandraDBCluster'\n\n\nseed_provider:\n  - class_name: org.apache.cassandra.locator.SimpleSeedProvider\n    parameters:\n         - seeds: \"112.522.6.61, 112.522.6.62, 112.522.6.63\"\n\n\n# ONLY THIS LINE CHANGES\nlisten_address: 112.522.6.62\n\n# ONLY THIS LINE CHANGES\nrpc_address: 112.522.6.62\n\n\nendpoint_snitch: GossipingPropertyFileSnitch\n\nauto_bootstrap: true\n</code></pre>"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#server-3-example-for-step-2","title":"Server 3 Example for Step 2","text":"<pre><code>cluster_name: 'CassandraDBCluster'\n\n\nseed_provider:\n  - class_name: org.apache.cassandra.locator.SimpleSeedProvider\n    parameters:\n         - seeds: \"112.522.6.61, 112.522.6.62, 112.522.6.63\"\n\n\n# ONLY THIS LINE CHANGES\nlisten_address: 112.522.6.63\n\n# ONLY THIS LINE CHANGES\nrpc_address: 112.522.6.63\n\n\nendpoint_snitch: GossipingPropertyFileSnitch\n\nauto_bootstrap: true\n</code></pre>"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#server-2-example-for-step-4","title":"Server 2 Example for Step 4","text":"<pre><code>sudo apt-get install iptables-persistent\n\n# ONLY THESE LINES CHANGE TO ALLOW COMMUNICATION WITH SERVER 1 AND 3\nsudo iptables -A INPUT -p tcp -s 112.522.6.61 -m multiport --dports 7000,9042 -m state --state NEW,ESTABLISHED -j ACCEPT\nsudo iptables -A INPUT -p tcp -s 112.522.6.63 -m multiport --dports 7000,9042 -m state --state NEW,ESTABLISHED -j ACCEPT\n\nsudo bash -c \"iptables-save &gt; /etc/iptables/rules.v4\"\n\nsudo netfilter-persistent reload\n\nsudo nodetool status\n</code></pre>"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#server-3-example-for-step-4","title":"Server 3 Example for Step 4","text":"<pre><code>sudo apt-get install iptables-persistent\n\n# ONLY THESE LINES CHANGE TO ALLOW COMMUNICATION WITH SERVER 1 AND 2\nsudo iptables -A INPUT -p tcp -s 112.522.6.61 -m multiport --dports 7000,9042 -m state --state NEW,ESTABLISHED -j ACCEPT\nsudo iptables -A INPUT -p tcp -s 112.522.6.62 -m multiport --dports 7000,9042 -m state --state NEW,ESTABLISHED -j ACCEPT\n\nsudo bash -c \"iptables-save &gt; /etc/iptables/rules.v4\"\n\nsudo netfilter-persistent reload\n\nsudo nodetool status\n</code></pre>"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#dead-node-fix","title":"Dead Node Fix","text":"<p>When your server/desktop restarts, you may face a dead node. You need replace the address with the same IP for it to work.</p> <pre><code>cd /etc/cassandra\nsubtl cassandra-env.sh\n</code></pre> <p>Add the following line in the last row assuming server 1 is dead where the IP is 112.522.6.61 <pre><code>JVM_OPTS=\"$JVM_OPTS -Dcassandra.replace_address=112.522.6.61\"\n</code></pre></p> <p>Then run the following bash commands</p> <pre><code>sudo rm -rf /var/lib/cassandra/data/system/*\nsudo service cassandra restart\nsudo nodetool status\n</code></pre> <p>You might have to wait awhile and re-run <code>sudo nodetool status</code> for the DB to get up and running.</p>"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#summary","title":"Summary","text":"<p>We have successfully set up a 3-node Cassandra cluster DB after all these steps. We will move on to interacting with the cluster with CQLSH and the Python Driver in subsequent guides.</p>"},{"location":"data_engineering/nosql/cassandra/setting_up_cluster/#citation","title":"Citation","text":"<p>If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.</p> <p> </p>"},{"location":"deep_learning/course_progression/","title":"Course Progression","text":"<p>If you would like a smooth transition in learning deep learning concepts, you need to follow the materials in a sequential order. Some sections are still pending as I am working on them, and they will have the  icon beside them.</p>"},{"location":"deep_learning/course_progression/#1-practical-deep-learning-with-pytorch","title":"1. Practical Deep Learning with PyTorch","text":"<ul> <li> Matrices</li> <li> Gradients</li> <li> Linear Regression</li> <li> Logistic Regression</li> <li> Feedforward Neural Networks (FNN)</li> <li> Convolutional Neural Networks (CNN)</li> <li> Recurrent Neural Networks (RNN)</li> <li> Long Short Term Memory Neural Networks (LSTM)</li> <li> Autoencoders (AE)</li> <li> Fully-connected Overcomplete Autoencoder (AE)</li> <li>  Variational Autoencoders (VAE)</li> <li>  Adversarial Autoencoders (AAE)</li> <li>  Generative Adversarial Networks (GAN)</li> <li>  Transformers</li> </ul>"},{"location":"deep_learning/course_progression/#2-improving-deep-learning-models-with-pytorch","title":"2. Improving Deep Learning Models with PyTorch","text":"<ul> <li> Derivatives, Gradients and Jacobian</li> <li> Gradient Descent and Backpropagation (From Scratch FNN Regression)</li> <li> Learning Rate Scheduling</li> <li> Optimizers</li> <li> Advanced Learning Rate Optimization</li> <li> Weight Initializations and Activation Functions</li> <li>  Overfitting Prevention</li> <li>  Loss, Accuracy and Weight Visualizations</li> <li>  Data Preprocessing for Images and Videos</li> <li>  Data Preprocessing for Time Series</li> </ul>"},{"location":"deep_learning/course_progression/#3-deep-reinforcement-learning-with-pytorch","title":"3. Deep Reinforcement Learning with PyTorch","text":"<ul> <li> Supervised Learning to Reinforcement Learning</li> <li> Markov Decision Processes and Bellman Equations</li> <li> Dynamic Programming</li> <li>  Monte Carlo Approach</li> <li>  Temporal-Difference</li> <li>  Policy Gradient: REINFORCE</li> <li>  Policy Gradient: Actor-Critic</li> <li>  Policy Gradient: A2C/A3C</li> <li>  Policy Gradient: ACKTR</li> <li>  Policy Gradient: PPO</li> <li>  Policy Gradient: DPG</li> <li>  Policy Gradient: DDPG (DQN &amp; DPG)</li> </ul>"},{"location":"deep_learning/course_progression/#4-from-scratch-with-python-and-pytorch","title":"4. From Scratch with Python and PyTorch","text":"<ul> <li> From Scratch Logistic Regression Classification</li> <li>  From Scratch FNN Classification</li> <li>  From Scratch CNN Classification</li> <li>  From Scratch RNN Classification</li> <li>  From Scratch LSTM Classification</li> <li>  From Scratch AE</li> </ul>"},{"location":"deep_learning/course_progression/#5-large-language-models-with-pytorch","title":"5. Large Language Models with PyTorch","text":"<ul> <li>  Introduction</li> <li>  What is Temperature</li> </ul>"},{"location":"deep_learning/intro/","title":"Deep Learning and Deep Reinforcement Learning Theory and Programming Tutorials","text":"<p>We'll be covering both CPU and GPU implementations of deep learning and deep reinforcement learning algorithms</p> Packages and Languages you will Learn to Use <ul> <li>Python</li> <li>PyTorch</li> <li>NumPy</li> <li>Gym</li> </ul> <p>If you would like a more visual and guided experience, feel free to take our video course.</p>"},{"location":"deep_learning/intro/#paths","title":"Paths","text":"<p>Work in progress</p> <p>This open-source portion is still a work in progress, it may be sparse in explanation as traditionally all our explanation are done via video. Stay tuned while we gradually upload our tutorials and notes. Feel free to contact Ritchie Ng if you would like to contribute via our Facebook page.</p> <p>Also take note that these notes are best used as a referral. This is because we have yet to expand it comprehensively to be a stand-alone guide. Go head and take our video course that provides a much easier, proven-to-work, experience.</p> <p>All of our code allows you to run in a notebook for this deep learning section. Please use a <code>jupyter notebook</code> and run the examples from the start of the page to the end.</p> <p>Remember to <code>Right mouse click &gt; Open image in new tab</code> if you would like to zoom into the diagrams if you find them too small.</p> <p>Clarifications</p> <p>These are some clarifications we would like to highlight. </p> <ul> <li> When we state linear function, more specifically we meant affine function that comprises a linear function and a constant. We did this initially to make it easier as \"linear function\" was easier to digest.</li> <li> For all diagrams that says valid padding, they refer to no padding such that your output size will be smaller than your input size.</li> <li> For all diagrams that says same padding, they refer to zero padding (padding your input with zeroes) such that your output size will be equal to your input size.</li> </ul> <p>Errors to be corrected</p> <p>As we are rapidly prototyping there may be some errors. For these errors stated here, they will be corrected soon.</p> <p>Feel free to report bugs, corrections or improvements on our Github repository. </p> <p>If you did not go through all the materials, you would not be familiar with these, so go through them and come back to review these changes.</p> <ul> <li> For all diagrams that says dot product, they refer to matrix product. </li> </ul>"},{"location":"deep_learning/readings/","title":"Readings","text":"<p>Info</p> <p>This is a list of growing number of papers and implementations I think are interesting.</p>"},{"location":"deep_learning/readings/#long-tailed-recognition","title":"Long Tailed Recognition","text":"<ul> <li>Large-Scale Long-Tailed Recognition in an Open World<ul> <li>Frequently in real world scenario there're new unseen classes or samples within the tail classes</li> <li>This tackles the problem with dynamic embedding to bring associative memory to aid prediction of long-tailed classes</li> <li>The model essentially combines direct image features with embeddings from other classes</li> </ul> </li> </ul>"},{"location":"deep_learning/readings/#better-generalization-overfitting-prevention-or-regularization","title":"Better Generalization (Overfitting Prevention or Regularization)","text":"<ul> <li>Domain Randomization for Transferring Deep Neural Networks from Simulation to the Real World<ul> <li>They propose to use domain randomization to train deep learning algorithms on synthetic data and transferring to real-world data</li> <li>The idea is that with sufficient variability in the textures of synthetic data, real-world data becomes another variation of the synthetic data</li> <li>It works surprisingly well and it's a simple technique of varying image textures essentially enabling CNNs to be more robust to variations in image textures</li> </ul> </li> </ul>"},{"location":"deep_learning/readings/#optimization","title":"Optimization","text":"<ul> <li>Online Learning Rate Adaptation with Hypergradient Descent<ul> <li>Reduces the need for learning rate scheduling for SGD, SGD and nesterov momentum, and Adam</li> <li>Uses the concept of hypergradients (gradients w.r.t. learning rate) obtained via reverse-mode automatic differentiation to dynamically update learning rates in real-time alongside weight updates</li> <li>Little additional computation because just needs just one additional copy of original gradients store in memory</li> <li>Severely under-appreciated paper</li> </ul> </li> </ul>"},{"location":"deep_learning/readings/#network-compression","title":"Network Compression","text":"<ul> <li>Energy-constrained Compression for Deep Neural Networks via Weighted Sparse Projection and Layer Input Masking<ul> <li>More production applications of DNN require low-energy consumption environment like self-driving cars, VR goggles, and drones</li> <li>As such it's critical to optimize DNN not for its primary performance (accuracy etc.) but for its energy consumption performance too </li> <li>In the DNN training, this paper introduces an energy budget constraint on top of other optimization objectives</li> <li>This allows optimization of multiple objectives simultaneously (top-1 accuracy and energy consumption for example)</li> <li>It's done through weighted sparse projection and layer input masking</li> </ul> </li> </ul>"},{"location":"deep_learning/readings/#architecture-search","title":"Architecture Search","text":"<ul> <li>DARTS: Differentiable Architecture Search<ul> <li>Neural search algorithm based on gradient descent and continuous relaxation in the architecture space. </li> <li>A good move towards automatic architecture designs of neural networks.</li> </ul> </li> <li>EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks<ul> <li>Scales all dimensions of a CNN, resolution/depth/width using compound coefficient</li> <li>Uses neural architecture search</li> </ul> </li> </ul>"},{"location":"deep_learning/readings/#network-pruning","title":"Network Pruning","text":"<ul> <li>EigenDamage: Structured Pruning in the Kronecker-Factored Eigenbasis<ul> <li>Compared to existing Hessian-based methods, this works on the KFE</li> <li>Reported 10x reduction in model size and 8x reduction in FLOPs on Wide ResNet32 (WRN32)</li> </ul> </li> </ul>"},{"location":"deep_learning/readings/#bayesian-deep-learning","title":"Bayesian Deep Learning","text":"<ul> <li>Fast and Scalable Bayesian Deep Learning by Weight-Perturbation in Adam<ul> <li>Variational Adam (Vadam), an alternative to varianal inference via dropout.</li> <li>Vadam perturbs the network's weights when backpropagating, allowing low computation cost uncertainty estimates. </li> <li>Not as good as dropout in terms of performance, but a good direction for computationally cheaper options.</li> </ul> </li> </ul>"},{"location":"deep_learning/readings/#explainability","title":"Explainability","text":"<ul> <li>A Unified Approach to Intepreting Model Predictions<ul> <li>Introduces SHAP (SHapley Additive exPlanations)</li> <li>\"SHAP assigns each feature an importance value for a particular prediction\"<ul> <li>Higher positive SHAP values (red) = increase the probability of the class</li> <li>Higher negative SHAP values (blue) = decrease the probability of the class</li> </ul> </li> </ul> </li> <li>Hierarchical interpretations for neural network predictions<ul> <li>Given a prediction from the deep neural network, agglomerative contextual decomposition (ACD) produces a hierarchical clusters of input features alongside cluster-wise contribution to the final prediction.</li> <li>The hierarchical clustering is then optimized to identify learned clusters driving the DNN's predictions.</li> </ul> </li> </ul>"},{"location":"deep_learning/readings/#cautious","title":"Cautious","text":"<ul> <li>Same Stats, Different Graphs: Generating Datasets with Varied Appearance and Identical Statistics through Simulated Annealing<ul> <li>Shows through scatterplots that multiple toy datasets although visually very different can have similar summary statistics like mean, standard deviation and pearson correlation</li> <li>This paper emphasises the need to always visualize your data</li> </ul> </li> </ul>"},{"location":"deep_learning/readings/#visualization","title":"Visualization","text":"<ul> <li>Netron<ul> <li>Easily visualize your saved deep learning models (PyTorch .pth, TensorFlow .pb, MXNet .model, ONNX, and more)</li> <li>You can even check out each node's documentation quickly in the interface</li> </ul> </li> </ul>"},{"location":"deep_learning/readings/#missing-values","title":"Missing Values","text":"<ul> <li>BRITS<ul> <li>If you face problems in missing data in your time series and you use existing imputation methods, there is an alternative called BRITS where it learns missing values in time series via a bidirectional recurrency dynamical system</li> </ul> </li> </ul>"},{"location":"deep_learning/readings/#correlation","title":"Correlation","text":"<ul> <li>DCCA: Deep Canonical Correlation Analysis<ul> <li>Learn non-linear complex transformations such that resulting transformed data have high linear correlation</li> <li>Alternative to non-parametric methods like kernel canonical correlation analysis (KCCA) and non-linear extension of canonical correlation analysis (CCA)</li> <li>Shown to learn higher correlation representations than CCA/KCCA</li> </ul> </li> </ul>"},{"location":"deep_learning/readings/#deep-reinforcement-learning","title":"Deep Reinforcement Learning","text":"<ul> <li>An Empirical Analysis of Proximal Policy Optimization with Kronecker-factored Natural Gradients<ul> <li>Shows 2 SOTA for deep RL currently (2018 / early 2019): PPO and ACKTR</li> <li>Attempts to combined PPO objective with K-FAC natural gradient optimization: PPOKFAC</li> <li>Does not improve sample complexity, stick with either PPO/ACKTR for now</li> </ul> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/","title":"Derivative, Gradient and Jacobian","text":"<p>Run Jupyter Notebook</p> <p>You can run the code for this section in this jupyter notebook link.</p>"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/#simplified-equation","title":"Simplified Equation","text":"<ul> <li>This is the simplified equation we have been using on how we update our parameters to reach good values (good local or global minima)</li> <li>\\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\)<ul> <li>\\(\\theta\\): parameters (our tensors with gradient accumulation abilities)</li> <li>\\(\\eta\\): learning rate (how fast we want to learn)</li> <li>\\(\\nabla_\\theta\\): gradients of loss with respect to the model's parameters</li> </ul> </li> <li>Even simplier equation in English: <code>parameters = parameters - learning_rate * parameters_gradients</code></li> <li>This process can be broken down into 2 sequential parts<ol> <li>Backpropagation</li> <li>Gradient descent</li> </ol> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/#simplified-equation-breakdown","title":"Simplified Equation Breakdown","text":"<ul> <li>Our simplified equation can be broken down into 2 parts<ol> <li>Backpropagation: getting our gradients<ul> <li>Our partial derivatives of loss (scalar number) with respect to (w.r.t.) our model's parameters and w.r.t. our input</li> <li>Backpropagation gets us \\(\\nabla_\\theta\\) which is our gradient</li> </ul> </li> <li>Gradient descent: using our gradients to update our parameters<ul> <li>Somehow, the terms backpropagation and gradient descent are often mixed together. But they're totally different. </li> <li>Gradient descent relates to using our gradients obtained from backpropagation to update our weights.</li> <li>Gradient descent: \\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\)</li> </ul> </li> </ol> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/#steps","title":"Steps","text":"<ul> <li>Derivatives</li> <li>Partial Derivatives</li> <li>Gradients</li> <li>Gradient, Jacobian and Generalized Jacobian Differences</li> <li>Backpropagation: computing gradients</li> <li>Gradient descent: using gradients to update parameters</li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/#derivative","title":"Derivative","text":"<ul> <li>Given a simple cubic equation: \\(f(x) = 2x^3 + 5\\)</li> <li>Calculating the derivative \\(\\frac{df(x)}{dx}\\) is simply calculating the difference in values of \\(y\\) for an extremely small (infinitesimally) change in value of \\(x\\) which is frequently labelled as \\(h\\)<ul> <li>\\(\\frac{df(x)}{dx} = \\displaystyle{\\lim_{h \\to 0}} \\frac{f(x + h) - f(x)}{h}\\)<ul> <li>\\(\\frac{f(x + h) - f(x)}{h}\\) is the slope formula similar to what you may be familiar with:<ul> <li>Change in \\(y\\) over change in \\(x\\): \\(\\frac{\\Delta y}{\\Delta x}\\)</li> </ul> </li> <li>And the derivative is the slope when \\(h \\rightarrow 0\\), in essence a super teeny small \\(h\\)</li> <li></li> </ul> </li> </ul> </li> <li>Let's break down \\(\\frac{df}{dx} = \\displaystyle{\\lim_{h \\to 0}} \\frac{f(x + h) - f(x)}{h}\\)<ul> <li>\\(\\displaystyle{\\lim_{h \\to 0} \\frac{f(x + h) - f(x)}{h}}\\)</li> <li>\\(\\displaystyle{\\lim_{h \\to 0} \\frac{(2(x+h)^3 + 5) - 2x^3 + 5}{h}}\\)</li> <li>\\(\\displaystyle{\\lim_{h \\to 0} \\frac{2(x^2 + 2xh + h^2)(x+h) - 2x^3}{h}}\\)</li> <li>\\(\\displaystyle{\\lim_{h \\to 0} \\frac{2(x^3 + 2x^2h + h^3 + x^2h + 2xh^2 + h^3) - 2x^3}{h}}\\)</li> <li>\\(\\displaystyle{\\lim_{h \\to 0}\\frac{2(x^3 + 3x^2h + h^3 + 2xh^2) - 2x^3}{h}}\\)</li> <li>\\(\\displaystyle{\\lim_{h \\to 0} \\frac{6x^2h + h^3 + 2xh^2}{h}}\\)</li> <li>\\(\\displaystyle{\\lim_{h \\to 0} 6x^2 + h^2 + 2xh} = 6x^2\\)</li> </ul> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/#partial-derivative","title":"Partial Derivative","text":"<ul> <li>Ok, it's simple to calculate our derivative when we've only one variable in our function. </li> <li>If we've more than one (as with our parameters in our models), we need to calculate our partial derivatives of our function with respect to our variables</li> <li>Given a simple equation \\(f(x, z) = 4x^4z^3\\), let us get our partial derivatives</li> <li>2 parts: partial derivative of our function w.r.t. x and z<ol> <li>Partial derivative of our function w.r.t. x: \\(\\frac{\\delta f(x, z)}{\\delta x}\\)<ul> <li>Let \\(z\\) term be a constant, \\(a\\) </li> <li>\\(f(x, z) = 4x^4a\\)</li> <li>\\(\\frac{\\delta f(x, z)}{\\delta x} = 16x^3a\\)</li> <li>Now we substitute \\(a\\) with our z term, \\(a = z^3\\)</li> <li>\\(\\frac{\\delta f(x, z)}{\\delta x} = 16x^3z^3\\)</li> </ul> </li> <li>Partial derivative of our function w.r.t. z: \\(\\frac{\\delta f(x, z)}{\\delta z}\\)<ul> <li>Let \\(x\\) term be a constant, \\(a\\)</li> <li>\\(f(x, z) = 4az^3\\)</li> <li>\\(\\frac{\\delta f(x, z)}{\\delta z} = 12az^2\\)</li> <li>Now we substitute \\(a\\) with our \\(x\\) term, \\(a = x^4\\)</li> <li>\\(\\frac{\\delta f(x, z)}{\\delta z} = 12x^4z^2\\)</li> </ul> </li> </ol> </li> <li>Ta da! We made it, we calculated our partial derivatives of our function w.r.t. the different variables</li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/#gradient","title":"Gradient","text":"<ul> <li>We can now put all our partial derivatives into a vector of partial derivatives<ul> <li>Also called \"gradient\"</li> <li>Represented by \\(\\nabla_{(x,z)}\\)</li> </ul> </li> <li>\\(\\nabla_{(x,z)} = \\begin{bmatrix} \\frac{df(x,z)}{dx} \\\\ \\frac{df(x,z)}{dz} \\end{bmatrix} = \\begin{bmatrix} 16x^3z^3 \\\\ 12x^4z^2 \\end{bmatrix}\\)</li> <li>It is critical to note that the term gradient applies for \\(f : \\mathbb{R}^N \\rightarrow \\mathbb{R}\\)<ul> <li>Where our function maps a vector input to a scalar output: in deep learning, our loss function that produces a scalar loss</li> </ul> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/#gradient-jacobian-and-generalized-jacobian","title":"Gradient, Jacobian, and Generalized Jacobian","text":"<p>In the case where we have non-scalar outputs, these are the right terms of matrices or vectors containing our partial derivatives</p> <ol> <li>Gradient: vector input to scalar output<ul> <li>\\(f : \\mathbb{R}^N \\rightarrow \\mathbb{R}\\)</li> </ul> </li> <li>Jacobian: vector input to vector output<ul> <li>\\(f : \\mathbb{R}^N \\rightarrow \\mathbb{R}^M\\)</li> </ul> </li> <li>Generalized Jacobian: tensor input to tensor output<ul> <li>In this case, a tensor can be any number of dimensions.</li> </ul> </li> </ol>"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/#summary","title":"Summary","text":"<p>We've learnt to...</p> <p>Success</p> <ul> <li> Calculate derivatives</li> <li> Calculate partial derivatives</li> <li> Get gradients</li> <li> Differentiate the concepts amongst gradients, Jacobian and Generalized Jacobian</li> </ul> <p>Now it is time to move on to backpropagation and gradient descent for a simple 1 hidden layer FNN with all these concepts in mind.</p>"},{"location":"deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/#citation","title":"Citation","text":"<p>If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.</p> <p> </p>"},{"location":"deep_learning/boosting_models_pytorch/forwardpropagation_backpropagation_gradientdescent/","title":"Forwardpropagation, Backpropagation and Gradient Descent with PyTorch","text":"<p>Run Jupyter Notebook</p> <p>You can run the code for this section in this jupyter notebook link.</p>"},{"location":"deep_learning/boosting_models_pytorch/forwardpropagation_backpropagation_gradientdescent/#transiting-to-backpropagation","title":"Transiting to Backpropagation","text":"<ul> <li>Let's go back to our simple FNN to put things in perspective<ul> <li>Let us ignore non-linearities for now to keep it simpler, but it's just a tiny change subsequently</li> <li>Given a linear transformation on our input (for simplicity instead of an affine transformation that includes a bias): \\(\\hat y = \\theta x\\)<ul> <li>\\(\\theta\\) is our parameters</li> <li>\\(x\\) is our input</li> <li>\\(\\hat y\\) is our prediction</li> </ul> </li> <li>Then we have our MSE loss function \\(L = \\frac{1}{2} (\\hat y - y)^2\\)</li> </ul> </li> <li>We need to calculate our partial derivatives of our loss w.r.t. our parameters to update our parameters: \\(\\nabla_{\\theta} = \\frac{\\delta L}{\\delta \\theta}\\)<ul> <li>With chain rule we have \\(\\frac{\\delta L}{\\delta \\theta} = \\frac{\\delta L}{\\delta \\hat y} \\frac{\\delta \\hat y}{\\delta \\theta}\\)<ul> <li>\\(\\frac{\\delta L}{\\delta \\hat y} = (\\hat y -  y)\\)</li> <li>\\(\\frac{\\delta \\hat y}{\\delta \\theta}\\) is our partial derivatives of \\(y\\) w.r.t. our parameters (our gradient) as we have covered previously</li> </ul> </li> </ul> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/forwardpropagation_backpropagation_gradientdescent/#forward-propagation-backward-propagation-and-gradient-descent","title":"Forward Propagation, Backward Propagation and Gradient Descent","text":"<ul> <li>All right, now let's put together what we have learnt on backpropagation and apply it on a simple feedforward neural network (FNN)</li> <li>Let us assume the following simple FNN architecture and take note that we do not have bias here to keep things simple<ul> <li>FNN architecture<ol> <li>Linear function: hidden size = 32</li> <li>Non-linear function: sigmoid</li> <li>Linear function: output size = 1</li> <li>Non-linear function: sigmoid</li> </ol> </li> <li>We will be going through a binary classification problem classifying 2 types of flowers<ul> <li>Output size: 1 (represented by 0 or 1 depending on the flower)</li> <li>Input size: 2 (features of the flower)</li> <li>Number of training samples: 100</li> </ul> </li> </ul> </li> </ul> <p>Load 3-class dataset</p> <p>We want to set a seed to encourage reproducibility so you can match our loss numbers.</p> <pre><code>import torch\nimport torch.nn as nn\n\n# Set manual seed\ntorch.manual_seed(2)\n</code></pre> <p>Here we want to load our flower classification dataset of 150 samples. There are 2 features, hence the input size would be 150x2. There is no one-hot encoding so the output would not be a size of 150x3 but a size of 150x1. <pre><code>from sklearn import datasets\nfrom sklearn import preprocessing\niris = datasets.load_iris()\nX = torch.tensor(preprocessing.normalize(iris.data[:, :2]), dtype=torch.float)\ny = torch.tensor(iris.target.reshape(-1, 1), dtype=torch.float)\n</code></pre></p> <pre><code>print(X.size())\nprint(y.size())\n</code></pre> <pre><code>torch.Size([150, 2])\ntorch.Size([150, 1])\n</code></pre> <p>From 3 class dataset to 2 class dataset</p> <p>We only want 2 classes because we want a binary classification problem. As mentioned, there is no one-hot encoding, so each class is represented by 0, 1, or 2. All we need to do is to filter out all samples with a label of 2 to have 2 classes. <pre><code># We only take 2 classes to make a binary classification problem\nX = X[:y[y &lt; 2].size()[0]]\ny = y[:y[y &lt; 2].size()[0]]\n````\n\n```python\nprint(X.size())\nprint(y.size())\n</code></pre></p> <pre><code>torch.Size([100, 2])\ntorch.Size([100, 1])\n</code></pre> <p>Building our FNN model class from scratch</p> <pre><code>class FNN(nn.Module):\n    def __init__(self, ):\n        super().__init__()\n\n        # Dimensions for input, hidden and output\n        self.input_dim = 2\n        self.hidden_dim = 32\n        self.output_dim = 1\n\n        # Learning rate definition\n        self.learning_rate = 0.001\n\n        # Our parameters (weights)\n        # w1: 2 x 32\n        self.w1 = torch.randn(self.input_dim, self.hidden_dim)\n\n        # w2: 32 x 1\n        self.w2 = torch.randn(self.hidden_dim, self.output_dim)\n\n    def sigmoid(self, s):\n        return 1 / (1 + torch.exp(-s))\n\n    def sigmoid_first_order_derivative(self, s):\n        return s * (1 - s)\n\n    # Forward propagation\n    def forward(self, X):\n        # First linear layer\n        self.y1 = torch.matmul(X, self.w1) # 3 X 3 \".dot\" does not broadcast in PyTorch\n\n        # First non-linearity\n        self.y2 = self.sigmoid(self.y1)\n\n        # Second linear layer\n        self.y3 = torch.matmul(self.y2, self.w2)\n\n        # Second non-linearity\n        y4 = self.sigmoid(self.y3)\n        return y4\n\n    # Backward propagation\n    def backward(self, X, l, y4):\n        # Derivative of binary cross entropy cost w.r.t. final output y4\n        self.dC_dy4 = y4 - l\n\n        '''\n        Gradients for w2: partial derivative of cost w.r.t. w2\n        dC/dw2\n        '''\n        self.dy4_dy3 = self.sigmoid_first_order_derivative(y4)\n        self.dy3_dw2 = self.y2\n\n        # Y4 delta: dC_dy4 dy4_dy3\n        self.y4_delta = self.dC_dy4 * self.dy4_dy3\n\n        # This is our gradients for w1: dC_dy4 dy4_dy3 dy3_dw2\n        self.dC_dw2 = torch.matmul(torch.t(self.dy3_dw2), self.y4_delta)\n\n        '''\n        Gradients for w1: partial derivative of cost w.r.t w1\n        dC/dw1\n        '''\n        self.dy3_dy2 = self.w2\n        self.dy2_dy1 = self.sigmoid_first_order_derivative(self.y2)\n\n        # Y2 delta: (dC_dy4 dy4_dy3) dy3_dy2 dy2_dy1\n        self.y2_delta = torch.matmul(self.y4_delta, torch.t(self.dy3_dy2)) * self.dy2_dy1\n\n        # Gradients for w1: (dC_dy4 dy4_dy3) dy3_dy2 dy2_dy1 dy1_dw1\n        self.dC_dw1 = torch.matmul(torch.t(X), self.y2_delta)\n\n        # Gradient descent on the weights from our 2 linear layers\n        self.w1 -= self.learning_rate * self.dC_dw1\n        self.w2 -= self.learning_rate * self.dC_dw2\n\n    def train(self, X, l):\n        # Forward propagation\n        y4 = self.forward(X)\n\n        # Backward propagation and gradient descent\n        self.backward(X, l, y4)\n</code></pre> <p>Training our FNN model</p> <pre><code># Instantiate our model class and assign it to our model object\nmodel = FNN()\n\n# Loss list for plotting of loss behaviour\nloss_lst = []\n\n# Number of times we want our FNN to look at all 100 samples we have, 100 implies looking through 100x\nnum_epochs = 101\n\n# Let's train our model with 100 epochs\nfor epoch in range(num_epochs):\n    # Get our predictions\n    y_hat = model(X)\n\n    # Cross entropy loss, remember this can never be negative by nature of the equation\n    # But it does not mean the loss can't be negative for other loss functions\n    cross_entropy_loss = -(y * torch.log(y_hat) + (1 - y) * torch.log(1 - y_hat))\n\n    # We have to take cross entropy loss over all our samples, 100 in this 2-class iris dataset\n    mean_cross_entropy_loss = torch.mean(cross_entropy_loss).detach().item()\n\n    # Print our mean cross entropy loss\n    if epoch % 20 == 0:\n        print('Epoch {} | Loss: {}'.format(epoch, mean_cross_entropy_loss))\n    loss_lst.append(mean_cross_entropy_loss)\n\n    # (1) Forward propagation: to get our predictions to pass to our cross entropy loss function\n    # (2) Back propagation: get our partial derivatives w.r.t. parameters (gradients)\n    # (3) Gradient Descent: update our weights with our gradients\n    model.train(X, y)\n</code></pre> <pre><code>Epoch 0 | Loss: 0.9228229522705078\nEpoch 20 | Loss: 0.6966760754585266\nEpoch 40 | Loss: 0.6714916229248047\nEpoch 60 | Loss: 0.6686137914657593\nEpoch 80 | Loss: 0.666690468788147\nEpoch 100 | Loss: 0.6648102402687073\n</code></pre> <p>Our loss is decreasing gradually, so it's learning. It has a possibility of reducing to almost 0 (overfitting) with sufficient model capacity (more layers or wider layers). We will explore overfitting and learning rate optimization subsequently.</p>"},{"location":"deep_learning/boosting_models_pytorch/forwardpropagation_backpropagation_gradientdescent/#summary","title":"Summary","text":"<p>We've learnt...</p> <p>Success</p> <ul> <li> The math behind forwardpropagation, backwardpropagation and gradient descent for FNN</li> <li> Implement a basic FNN from scratch with PyTorch</li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/forwardpropagation_backpropagation_gradientdescent/#citation","title":"Citation","text":"<p>If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.</p> <p> </p>"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/","title":"Learning Rate Scheduling","text":"<p>Run Jupyter Notebook</p> <p>You can run the code for this section in this jupyter notebook link.</p>"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#optimization-algorithm-mini-batch-stochastic-gradient-descent-sgd","title":"Optimization Algorithm: Mini-batch Stochastic Gradient Descent (SGD)","text":"<ul> <li>We will be using mini-batch gradient descent in all our examples here when scheduling our learning rate</li> <li>Combination of batch gradient descent &amp; stochastic gradient descent<ul> <li>\\(\\theta = \\theta - \\eta \\cdot  \\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\)</li> </ul> </li> <li>Characteristics<ul> <li>Compute the gradient of the lost function w.r.t. parameters for n sets of training sample (n input and n label), \\(\\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\)</li> <li>Use this to update our parameters at every iteration</li> </ul> </li> <li>Typically in deep learning, some variation of mini-batch gradient is used where the batch size is a hyperparameter to be determined</li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#learning-intuition-recap","title":"Learning Intuition Recap","text":"<ul> <li>Learning process<ul> <li>Original parameters \\(\\rightarrow\\) given input, get output \\(\\rightarrow\\) compare with labels \\(\\rightarrow\\) get loss with comparison of input/output \\(\\rightarrow\\) get gradients of loss w.r.t parameters \\(\\rightarrow\\) update parameters so model can churn output closer to labels \\(\\rightarrow\\) repeat</li> </ul> </li> <li>For a detailed mathematical account of how this works and how to implement from scratch in Python and PyTorch, you can read our forward- and back-propagation and gradient descent post.</li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#learning-rate-pointers","title":"Learning Rate Pointers","text":"<ul> <li>Update parameters so model can churn output closer to labels, lower loss<ul> <li>\\(\\theta = \\theta - \\eta \\cdot  \\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\)</li> </ul> </li> <li>If we set \\(\\eta\\) to be a large value \\(\\rightarrow\\) learn too much (rapid learning)<ul> <li>Unable to converge to a good local minima (unable to effectively gradually decrease your loss, overshoot the local lowest value)</li> </ul> </li> <li>If we set \\(\\eta\\) to be a small value \\(\\rightarrow\\) learn too little (slow learning)<ul> <li>May take too long or unable to converge to a good local minima</li> </ul> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#need-for-learning-rate-schedules","title":"Need for Learning Rate Schedules","text":"<ul> <li>Benefits<ul> <li>Converge faster</li> <li>Higher accuracy </li> </ul> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#top-basic-learning-rate-schedules","title":"Top Basic Learning Rate Schedules","text":"<ol> <li>Step-wise Decay </li> <li>Reduce on Loss Plateau Decay</li> </ol>"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#step-wise-learning-rate-decay","title":"Step-wise Learning Rate Decay","text":""},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#step-wise-decay-every-epoch","title":"Step-wise Decay: Every Epoch","text":"<ul> <li>At every epoch,<ul> <li>\\(\\eta_t = \\eta_{t-1}\\gamma\\)</li> <li>\\(\\gamma = 0.1\\)</li> </ul> </li> <li>Optimization Algorithm 4: SGD Nesterov<ul> <li>Modification of SGD Momentum <ul> <li>\\(v_t = \\gamma v_{t-1} + \\eta \\cdot  \\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\)</li> <li>\\(\\theta = \\theta - v_t\\)</li> </ul> </li> </ul> </li> <li>Practical example<ul> <li>Given \\(\\eta_t = 0.1\\) and $ \\gamma = 0.01$</li> <li>Epoch 0: \\(\\eta_t = 0.1\\)</li> <li>Epoch 1: \\(\\eta_{t+1} = 0.1 (0.1) =  0.01\\)</li> <li>Epoch 2: \\(\\eta_{t+2} = 0.1 (0.1)^2 =  0.001\\)</li> <li>Epoch n: \\(\\eta_{t+n} = 0.1 (0.1)^n\\)</li> </ul> </li> </ul> <p>Code for step-wise learning rate decay at every epoch</p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n# Set seed\ntorch.manual_seed(0)\n\n# Where to add a new import\nfrom torch.optim.lr_scheduler import StepLR\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Non-linearity\n        self.relu = nn.ReLU()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.relu(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)\n\n'''\nSTEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS\n'''\n# step_size: at how many multiples of epoch you decay\n# step_size = 1, after every 1 epoch, new_lr = lr*gamma \n# step_size = 2, after every 2 epoch, new_lr = lr*gamma \n\n# gamma = decaying factor\nscheduler = StepLR(optimizer, step_size=1, gamma=0.1)\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    # Decay Learning Rate\n    scheduler.step()\n    # Print Learning Rate\n    print('Epoch:', epoch,'LR:', scheduler.get_lr())\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, 28*28)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Epoch: 0 LR: [0.1]\nIteration: 500. Loss: 0.15292978286743164. Accuracy: 96\nEpoch: 1 LR: [0.010000000000000002]\nIteration: 1000. Loss: 0.1207798570394516. Accuracy: 97\nEpoch: 2 LR: [0.0010000000000000002]\nIteration: 1500. Loss: 0.12287932634353638. Accuracy: 97\nEpoch: 3 LR: [0.00010000000000000003]\nIteration: 2000. Loss: 0.05614742264151573. Accuracy: 97\nEpoch: 4 LR: [1.0000000000000003e-05]\nIteration: 2500. Loss: 0.06775809079408646. Accuracy: 97\nIteration: 3000. Loss: 0.03737065941095352. Accuracy: 97\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#step-wise-decay-every-2-epochs","title":"Step-wise Decay: Every 2 Epochs","text":"<ul> <li>At every 2 epoch,<ul> <li>\\(\\eta_t = \\eta_{t-1}\\gamma\\)</li> <li>\\(\\gamma = 0.1\\)</li> </ul> </li> <li>Optimization Algorithm 4: SGD Nesterov<ul> <li>Modification of SGD Momentum <ul> <li>\\(v_t = \\gamma v_{t-1} + \\eta \\cdot  \\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\)</li> <li>\\(\\theta = \\theta - v_t\\)</li> </ul> </li> </ul> </li> <li>Practical example<ul> <li>Given \\(\\eta_t = 0.1\\) and \\(\\gamma = 0.01\\)</li> <li>Epoch 0: \\(\\eta_t = 0.1\\)</li> <li>Epoch 1: \\(\\eta_{t+1} = 0.1\\)</li> <li>Epoch 2: \\(\\eta_{t+2} = 0.1 (0.1) =  0.01\\)</li> </ul> </li> </ul> <p>Code for step-wise learning rate decay at every 2 epoch</p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n# Set seed\ntorch.manual_seed(0)\n\n# Where to add a new import\nfrom torch.optim.lr_scheduler import StepLR\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Non-linearity\n        self.relu = nn.ReLU()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.relu(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)\n\n'''\nSTEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS\n'''\n# step_size: at how many multiples of epoch you decay\n# step_size = 1, after every 2 epoch, new_lr = lr*gamma \n# step_size = 2, after every 2 epoch, new_lr = lr*gamma \n\n# gamma = decaying factor\nscheduler = StepLR(optimizer, step_size=2, gamma=0.1)\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    # Decay Learning Rate\n    scheduler.step()\n    # Print Learning Rate\n    print('Epoch:', epoch,'LR:', scheduler.get_lr())\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as Variable\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, 28*28).requires_grad_()\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Epoch: 0 LR: [0.1]\nIteration: 500. Loss: 0.15292978286743164. Accuracy: 96\nEpoch: 1 LR: [0.1]\nIteration: 1000. Loss: 0.11253029108047485. Accuracy: 96\nEpoch: 2 LR: [0.010000000000000002]\nIteration: 1500. Loss: 0.14498558640480042. Accuracy: 97\nEpoch: 3 LR: [0.010000000000000002]\nIteration: 2000. Loss: 0.03691177815198898. Accuracy: 97\nEpoch: 4 LR: [0.0010000000000000002]\nIteration: 2500. Loss: 0.03511016443371773. Accuracy: 97\nIteration: 3000. Loss: 0.029424520209431648. Accuracy: 97\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#step-wise-decay-every-epoch-larger-gamma","title":"Step-wise Decay: Every Epoch, Larger Gamma","text":"<ul> <li>At every epoch,<ul> <li>\\(\\eta_t = \\eta_{t-1}\\gamma\\)</li> <li>\\(\\gamma = 0.96\\)</li> </ul> </li> <li>Optimization Algorithm 4: SGD Nesterov<ul> <li>Modification of SGD Momentum <ul> <li>\\(v_t = \\gamma v_{t-1} + \\eta \\cdot  \\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\)</li> <li>\\(\\theta = \\theta - v_t\\)</li> </ul> </li> </ul> </li> <li>Practical example<ul> <li>Given \\(\\eta_t = 0.1\\) and \\(\\gamma = 0.96\\)</li> <li>Epoch 1: \\(\\eta_t = 0.1\\)</li> <li>Epoch 2: \\(\\eta_{t+1} = 0.1 (0.96) =  0.096\\)</li> <li>Epoch 3: \\(\\eta_{t+2} = 0.1 (0.96)^2 =  0.092\\)</li> <li>Epoch n: \\(\\eta_{t+n} = 0.1 (0.96)^n\\)</li> </ul> </li> </ul> <p>Code for step-wise learning rate decay at every epoch with larger gamma</p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n# Set seed\ntorch.manual_seed(0)\n\n# Where to add a new import\nfrom torch.optim.lr_scheduler import StepLR\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Non-linearity\n        self.relu = nn.ReLU()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.relu(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)\n\n'''\nSTEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS\n'''\n# step_size: at how many multiples of epoch you decay\n# step_size = 1, after every 2 epoch, new_lr = lr*gamma \n# step_size = 2, after every 2 epoch, new_lr = lr*gamma \n\n# gamma = decaying factor\nscheduler = StepLR(optimizer, step_size=2, gamma=0.96)\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    # Decay Learning Rate\n    scheduler.step()\n    # Print Learning Rate\n    print('Epoch:', epoch,'LR:', scheduler.get_lr())\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as Variable\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, 28*28)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Epoch: 0 LR: [0.1]\nIteration: 500. Loss: 0.15292978286743164. Accuracy: 96\nEpoch: 1 LR: [0.1]\nIteration: 1000. Loss: 0.11253029108047485. Accuracy: 96\nEpoch: 2 LR: [0.096]\nIteration: 1500. Loss: 0.11864850670099258. Accuracy: 97\nEpoch: 3 LR: [0.096]\nIteration: 2000. Loss: 0.030942382290959358. Accuracy: 97\nEpoch: 4 LR: [0.09216]\nIteration: 2500. Loss: 0.04521659016609192. Accuracy: 97\nIteration: 3000. Loss: 0.027839098125696182. Accuracy: 97\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#pointers-on-step-wise-decay","title":"Pointers on Step-wise Decay","text":"<ul> <li>You would want to decay your LR gradually when you're training more epochs<ul> <li>Converge too fast, to a crappy loss/accuracy, if you decay rapidly</li> </ul> </li> <li>To decay slower<ul> <li>Larger \\(\\gamma\\)</li> <li>Larger interval of decay</li> </ul> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#reduce-on-loss-plateau-decay","title":"Reduce on Loss Plateau Decay","text":""},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#reduce-on-loss-plateau-decay-patience0-factor01","title":"Reduce on Loss Plateau Decay, Patience=0, Factor=0.1","text":"<ul> <li>Reduce learning rate whenever loss plateaus<ul> <li>Patience: number of epochs with no improvement after which learning rate will be reduced<ul> <li>Patience = 0</li> </ul> </li> <li>Factor: multiplier to decrease learning rate, \\(lr = lr*factor = \\gamma\\)<ul> <li>Factor = 0.1</li> </ul> </li> </ul> </li> <li>Optimization Algorithm: SGD Nesterov<ul> <li>Modification of SGD Momentum <ul> <li>\\(v_t = \\gamma v_{t-1} + \\eta \\cdot  \\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\)</li> <li>\\(\\theta = \\theta - v_t\\)</li> </ul> </li> </ul> </li> </ul> <p>Code for reduce on loss plateau learning rate decay of factor 0.1 and 0 patience</p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n# Set seed\ntorch.manual_seed(0)\n\n# Where to add a new import\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 6000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Non-linearity\n        self.relu = nn.ReLU()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.relu(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)\n\n'''\nSTEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS\n'''\n# lr = lr * factor \n# mode='max': look for the maximum validation accuracy to track\n# patience: number of epochs - 1 where loss plateaus before decreasing LR\n        # patience = 0, after 1 bad epoch, reduce LR\n# factor = decaying factor\nscheduler = ReduceLROnPlateau(optimizer, mode='max', factor=0.1, patience=0, verbose=True)\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as Variable\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, 28*28)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                # Without .item(), it is a uint8 tensor which will not work when you pass this number to the scheduler\n                correct += (predicted == labels).sum().item()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            # print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))\n\n    # Decay Learning Rate, pass validation accuracy for tracking at every epoch\n    print('Epoch {} completed'.format(epoch))\n    print('Loss: {}. Accuracy: {}'.format(loss.item(), accuracy))\n    print('-'*20)\n    scheduler.step(accuracy)\n</code></pre> <pre><code>Epoch 0 completed\nLoss: 0.17087846994400024. Accuracy: 96.26\n--------------------\nEpoch 1 completed\nLoss: 0.11688263714313507. Accuracy: 96.96\n--------------------\nEpoch 2 completed\nLoss: 0.035437121987342834. Accuracy: 96.78\n--------------------\nEpoch     2: reducing learning rate of group 0 to 1.0000e-02.\nEpoch 3 completed\nLoss: 0.0324370414018631. Accuracy: 97.7\n--------------------\nEpoch 4 completed\nLoss: 0.022194599732756615. Accuracy: 98.02\n--------------------\nEpoch 5 completed\nLoss: 0.007145566865801811. Accuracy: 98.03\n--------------------\nEpoch 6 completed\nLoss: 0.01673538237810135. Accuracy: 98.05\n--------------------\nEpoch 7 completed\nLoss: 0.025424446910619736. Accuracy: 98.01\n--------------------\nEpoch     7: reducing learning rate of group 0 to 1.0000e-03.\nEpoch 8 completed\nLoss: 0.014696130529046059. Accuracy: 98.05\n--------------------\nEpoch     8: reducing learning rate of group 0 to 1.0000e-04.\nEpoch 9 completed\nLoss: 0.00573748117312789. Accuracy: 98.04\n--------------------\nEpoch     9: reducing learning rate of group 0 to 1.0000e-05.\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#reduce-on-loss-plateau-decay-patience0-factor05","title":"Reduce on Loss Plateau Decay, Patience=0, Factor=0.5","text":"<ul> <li>Reduce learning rate whenever loss plateaus<ul> <li>Patience: number of epochs with no improvement after which learning rate will be reduced<ul> <li>Patience = 0</li> </ul> </li> <li>Factor: multiplier to decrease learning rate, \\(lr = lr*factor = \\gamma\\)<ul> <li>Factor = 0.5</li> </ul> </li> </ul> </li> <li>Optimization Algorithm 4: SGD Nesterov<ul> <li>Modification of SGD Momentum <ul> <li>\\(v_t = \\gamma v_{t-1} + \\eta \\cdot  \\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\)</li> <li>\\(\\theta = \\theta - v_t\\)</li> </ul> </li> </ul> </li> </ul> <p>Code for reduce on loss plateau learning rate decay with factor 0.5 and 0 patience</p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n# Set seed\ntorch.manual_seed(0)\n\n# Where to add a new import\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 6000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Non-linearity\n        self.relu = nn.ReLU()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.relu(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)\n\n'''\nSTEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS\n'''\n# lr = lr * factor \n# mode='max': look for the maximum validation accuracy to track\n# patience: number of epochs - 1 where loss plateaus before decreasing LR\n        # patience = 0, after 1 bad epoch, reduce LR\n# factor = decaying factor\nscheduler = ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=0, verbose=True)\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as Variable\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, 28*28)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                 # Without .item(), it is a uint8 tensor which will not work when you pass this number to the scheduler\n                correct += (predicted == labels).sum().item()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            # print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))\n\n    # Decay Learning Rate, pass validation accuracy for tracking at every epoch\n    print('Epoch {} completed'.format(epoch))\n    print('Loss: {}. Accuracy: {}'.format(loss.item(), accuracy))\n    print('-'*20)\n    scheduler.step(accuracy)\n</code></pre> <pre><code>Epoch 0 completed\nLoss: 0.17087846994400024. Accuracy: 96.26\n--------------------\nEpoch 1 completed\nLoss: 0.11688263714313507. Accuracy: 96.96\n--------------------\nEpoch 2 completed\nLoss: 0.035437121987342834. Accuracy: 96.78\n--------------------\nEpoch     2: reducing learning rate of group 0 to 5.0000e-02.\nEpoch 3 completed\nLoss: 0.04893001914024353. Accuracy: 97.62\n--------------------\nEpoch 4 completed\nLoss: 0.020584167912602425. Accuracy: 97.86\n--------------------\nEpoch 5 completed\nLoss: 0.006022400688380003. Accuracy: 97.95\n--------------------\nEpoch 6 completed\nLoss: 0.028374142944812775. Accuracy: 97.87\n--------------------\nEpoch     6: reducing learning rate of group 0 to 2.5000e-02.\nEpoch 7 completed\nLoss: 0.013204765506088734. Accuracy: 98.0\n--------------------\nEpoch 8 completed\nLoss: 0.010137186385691166. Accuracy: 97.95\n--------------------\nEpoch     8: reducing learning rate of group 0 to 1.2500e-02.\nEpoch 9 completed\nLoss: 0.0035198689438402653. Accuracy: 98.01\n--------------------\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#pointers-on-reduce-on-loss-pleateau-decay","title":"Pointers on Reduce on Loss Pleateau Decay","text":"<ul> <li>In these examples, we used patience=1 because we are running few epochs<ul> <li>You should look at a larger patience such as 5 if for example you ran 500 epochs. </li> </ul> </li> <li>You should experiment with 2 properties <ul> <li>Patience</li> <li>Decay factor </li> </ul> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#summary","title":"Summary","text":"<p>We've learnt...</p> <p>Success</p> <ul> <li> Learning Rate Intuition<ul> <li> Update parameters so model can churn output closer to labels</li> <li> Gradual parameter updates</li> </ul> </li> <li> Learning Rate Pointers<ul> <li> If we set \\(\\eta\\) to be a large value \\(\\rightarrow\\) learn too much (rapid learning)</li> <li> If we set \\(\\eta\\) to be a small value \\(\\rightarrow\\) learn too little (slow learning)</li> </ul> </li> <li> Learning Rate Schedules<ul> <li> Step-wise Decay</li> <li> Reduce on Loss Plateau Decay</li> </ul> </li> <li> Step-wise Decay<ul> <li> Every 1 epoch</li> <li> Every 2 epoch</li> <li> Every 1 epoch, larger gamma</li> </ul> </li> <li> Step-wise Decay Pointers<ul> <li> Decay LR gradually<ul> <li> Larger \\(\\gamma\\)</li> <li> Larger interval of decay (increase epoch)</li> </ul> </li> </ul> </li> <li> Reduce on Loss Plateau Decay<ul> <li> Patience=0, Factor=1</li> <li> Patience=0, Factor=0.5</li> </ul> </li> <li> Pointers on Reduce on Loss Plateau Decay<ul> <li> Larger patience with more epochs</li> <li> 2 hyperparameters to experiment<ul> <li> Patience</li> <li> Decay factor</li> </ul> </li> </ul> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/lr_scheduling/#citation","title":"Citation","text":"<p>If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.</p> <p> </p>"},{"location":"deep_learning/boosting_models_pytorch/optimizers/","title":"Optimization Algorithms","text":"<p>Run Jupyter Notebook</p> <p>You can run the code for this section in this jupyter notebook link.</p>"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#introduction-to-gradient-descent-optimizers","title":"Introduction to Gradient-descent Optimizers","text":""},{"location":"deep_learning/boosting_models_pytorch/optimizers/#model-recap-1-hidden-layer-feedforward-neural-network-relu-activation","title":"Model Recap: 1 Hidden Layer Feedforward Neural Network (ReLU Activation)","text":""},{"location":"deep_learning/boosting_models_pytorch/optimizers/#steps","title":"Steps","text":"<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> </ul> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n# Set seed\ntorch.manual_seed(0)\n\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Non-linearity\n        self.relu = nn.ReLU()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.relu(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as Variable\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, 28*28)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 0.3440718352794647. Accuracy: 91\nIteration: 1000. Loss: 0.2057694047689438. Accuracy: 93\nIteration: 1500. Loss: 0.2646750807762146. Accuracy: 94\nIteration: 2000. Loss: 0.17563636600971222. Accuracy: 94\nIteration: 2500. Loss: 0.1361844837665558. Accuracy: 95\nIteration: 3000. Loss: 0.11089023947715759. Accuracy: 95\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#non-technical-process","title":"Non-Technical Process","text":"<ol> <li>Convert inputs/labels to variables</li> <li>Clear gradient buffers</li> <li>Get output given inputs </li> <li>Get loss by comparing with labels</li> <li>Get gradients w.r.t. parameters (backpropagation)</li> <li>Update parameters using gradients (gradient descent)<ul> <li><code>parameters = parameters - learning_rate * parameters_gradients</code></li> </ul> </li> <li>REPEAT</li> </ol>"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#why-is-it-called-gradient-descent","title":"Why is it called Gradient Descent?","text":"<ul> <li>Use gradients (calculated through backpropagation) \\(\\rightarrow\\) update parameters to minimize our loss (descent) \\(\\rightarrow\\) better predictive accuracy</li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#mathematical-interpretation-of-gradient-descent","title":"Mathematical Interpretation of Gradient Descent","text":"<ul> <li>Model's parameters: \\(\\theta \\in \u211d^d\\)</li> <li>Loss function: \\(J(\\theta)\\)</li> <li>Gradient w.r.t. parameters: \\(\\nabla J(\\theta)\\)</li> <li>Learning rate: \\(\\eta\\)</li> <li>Batch Gradient descent: \\(\\theta = \\theta - \\eta \\cdot  \\nabla J(\\theta)\\)</li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithm-1-batch-gradient-descent","title":"Optimization Algorithm 1: Batch Gradient Descent","text":"<ul> <li>What we've covered so far: batch gradient descent<ul> <li>\\(\\theta = \\theta - \\eta \\cdot  \\nabla J(\\theta)\\)</li> </ul> </li> <li>Characteristics<ul> <li>Compute the gradient of the lost function w.r.t. parameters for the entire training data, \\(\\nabla J(\\theta)\\) </li> <li>Use this to update our parameters at every iteration</li> </ul> </li> <li>Problems<ul> <li>Unable to fit whole datasets in memory </li> <li>Computationally slow as we attempt to compute a large gradient matrix \\(\\rightarrow\\) first order derivative, \\(\\nabla J(\\theta)\\)</li> </ul> </li> <li>Conceptually easy to understand \\(\\rightarrow\\) rarely used</li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithm-2-stochastic-gradient-descent","title":"Optimization Algorithm 2: Stochastic Gradient Descent","text":"<ul> <li>Modification of batch gradient descent<ul> <li>\\(\\theta = \\theta - \\eta \\cdot  \\nabla J(\\theta, x^{i}, y^{i})\\)</li> </ul> </li> <li>Characteristics<ul> <li>Compute the gradient of the lost function w.r.t. parameters for the one set of training sample (1 input and 1 label), \\(\\nabla J(\\theta, x^{i}, y^{i})\\)</li> <li>Use this to update our parameters at every iteration</li> </ul> </li> <li>Benefits<ul> <li>Able to fit large datasets</li> <li>Computationally faster \\(\\rightarrow\\) instead gradients w.r.t to the whole training data, we get the gradients w.r.t. training sample</li> </ul> </li> <li>Problems<ul> <li>Updating very frequently \\(\\rightarrow\\) huge variance in parameter updates \\(\\rightarrow\\) may overshoot local minima <ul> <li>Can be solved by carefully decaying your learning rate \\(\\rightarrow\\) take smaller steps in incorporating gradients to improve the parameters</li> </ul> </li> </ul> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithm-3-mini-batch-gradient-descent","title":"Optimization Algorithm 3: Mini-batch Gradient Descent","text":"<ul> <li>Combination of batch gradient descent &amp; stochastic gradient descent<ul> <li>\\(\\theta = \\theta - \\eta \\cdot  \\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\)</li> </ul> </li> <li>Characteristics<ul> <li>Compute the gradient of the lost function w.r.t. parameters for n sets of training sample (n input and n label), \\(\\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\)</li> <li>Use this to update our parameters at every iteration</li> </ul> </li> <li>Benefits<ul> <li>Able to fit large datasets</li> <li>Computationally faster \\(\\rightarrow\\) instead gradients w.r.t to the whole training data, we get the gradients w.r.t. training sample</li> <li>Lower variance of parameter updates</li> </ul> </li> <li>This is often called SGD in deep learning frameworks .__. </li> </ul> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n# Set seed\ntorch.manual_seed(0)\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim)\n        # Non-linearity\n        self.relu = nn.ReLU()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.relu(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as Variable\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, 28*28).requires_grad_()\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 0.3440718352794647. Accuracy: 91\nIteration: 1000. Loss: 0.2057694047689438. Accuracy: 93\nIteration: 1500. Loss: 0.2646750807762146. Accuracy: 94\nIteration: 2000. Loss: 0.17563636600971222. Accuracy: 94\nIteration: 2500. Loss: 0.1361844837665558. Accuracy: 95\nIteration: 3000. Loss: 0.11089023947715759. Accuracy: 95\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithm-4-sgd-momentum","title":"Optimization Algorithm 4: SGD Momentum","text":"<ul> <li>Modification of SGD<ul> <li>\\(v_t = \\gamma v_{t-1} + \\eta \\cdot  \\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\)</li> <li>\\(\\theta = \\theta - v_t\\)</li> </ul> </li> <li>Characteristics<ul> <li>Compute the gradient of the lost function w.r.t. parameters for n sets of training sample (n input and n label), \\(\\nabla J(\\theta, x^{i: i+n}, y^{i:i+n})\\)</li> <li>Use this to add to the previous update vector \\(v_{t-1}\\)</li> <li>Momentum, usually set to \\(\\gamma = 0.9\\)</li> <li>Parameters updated with update vector, \\(v_t\\) that incorporates previous update vector<ul> <li>\\(\\gamma v_{t}\\) increases if gradient same sign/direction as \\(v_{t-1}\\) <ul> <li>Gives SGD the push when it is going in the right direction (minimizing loss)</li> <li>Accelerated convergence</li> </ul> </li> <li>\\(\\gamma v_{t}\\) decreases if gradient different sign/direction as \\(v_{t-1}\\)<ul> <li>Dampens SGD when it is going in a different direction</li> <li>Lower variation in loss minimization</li> </ul> </li> </ul> </li> </ul> </li> <li>Problems<ul> <li>It might go the wrong direction (higher loss) \\(\\rightarrow\\) continue to be accelerated to the wrong direction (higher loss) </li> </ul> </li> </ul> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n# Set seed\ntorch.manual_seed(0)\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Non-linearity\n        self.relu = nn.ReLU()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.relu(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9)\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as Variable\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, 28*28)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 0.16120098531246185. Accuracy: 96\nIteration: 1000. Loss: 0.15727552771568298. Accuracy: 96\nIteration: 1500. Loss: 0.1303034871816635. Accuracy: 96\nIteration: 2000. Loss: 0.022178759798407555. Accuracy: 97\nIteration: 2500. Loss: 0.07027597725391388. Accuracy: 97\nIteration: 3000. Loss: 0.02519878000020981. Accuracy: 97\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithm-4-sgd-nesterov","title":"Optimization Algorithm 4: SGD Nesterov","text":"<ul> <li>Modification of SGD Momentum <ul> <li>\\(v_t = \\gamma v_{t-1} + \\eta \\cdot  \\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\)</li> <li>\\(\\theta = \\theta - v_t\\)</li> </ul> </li> <li>Characteristics<ul> <li>Compute the gradient of the lost function w.r.t. future approximate parameters for n sets of training sample (n input and n label), \\(\\nabla J(\\theta - \\gamma v_{t-1}, x^{i: i+n}, y^{i:i+n})\\)<ul> <li>Use this to add to the previous update vector \\(v_{t-1}\\)</li> <li>Momentum, usually set to \\(\\gamma = 0.9\\)</li> </ul> </li> <li>Gradients w.r.t. future approximate parameters \\(\\rightarrow\\) sense of where we will be \\(\\rightarrow\\) anticipate if we are going in the wrong direction in the next step \\(\\rightarrow\\) slow down accordingly</li> </ul> </li> </ul> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n# Set seed\ntorch.manual_seed(0)\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Non-linearity\n        self.relu = nn.ReLU()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.relu(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as Variable\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, 28*28)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 0.15292978286743164. Accuracy: 96\nIteration: 1000. Loss: 0.11253029108047485. Accuracy: 96\nIteration: 1500. Loss: 0.11986596137285233. Accuracy: 96\nIteration: 2000. Loss: 0.016192540526390076. Accuracy: 97\nIteration: 2500. Loss: 0.06744947284460068. Accuracy: 97\nIteration: 3000. Loss: 0.03692319989204407. Accuracy: 97\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithm-4-adam","title":"Optimization Algorithm 4: Adam","text":"<ul> <li>Adaptive Learning Rates<ul> <li>\\(m_t = \\beta_1 m_{t-1} + (1 - \\beta_1)g_t\\)<ul> <li>Keeping track of decaying gradient</li> <li>Estimate of the mean of gradients</li> </ul> </li> <li>\\(v_t = \\beta_2 v_{t-1} + (1 - \\beta_2)g_t^2\\)<ul> <li>Keeping track of decaying squared gradient </li> <li>Estimate of the variance of gradients</li> </ul> </li> <li>When \\(m_t, v_t\\) initializes as 0, \\(m_t, v_t \\rightarrow 0\\) initially when decay rates small, \\(\\beta_1, \\beta_2 \\rightarrow 1\\) <ul> <li>Need to correct this with:</li> <li>\\(\\hat m_t = \\frac{m_t}{1- \\beta_1}\\)</li> <li>\\(\\hat v_t = \\frac{v_t}{1- \\beta_2}\\)</li> </ul> </li> <li>\\(\\theta_{t+1} = \\theta_t - \\frac{\\eta}{\\sqrt{\\hat v_t} + \\epsilon}\\hat m_t\\)</li> <li>Default recommended values<ul> <li>\\(\\beta_1 = 0.9\\)</li> <li>\\(\\beta_2 = 0.999\\)</li> <li>\\(\\epsilon = 10^{-8}\\)</li> </ul> </li> </ul> </li> <li>Instead of learning rate \\(\\rightarrow\\) equations account for estimates of mean/variance of gradients to determine the next learning rate</li> </ul> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n# Set seed\ntorch.manual_seed(0)\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Non-linearity\n        self.relu = nn.ReLU()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.relu(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\n# learning_rate = 0.001\n\noptimizer = torch.optim.Adam(model.parameters())\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as Variable\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, 28*28)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 0.2703690826892853. Accuracy: 93\nIteration: 1000. Loss: 0.15547044575214386. Accuracy: 95\nIteration: 1500. Loss: 0.17266806960105896. Accuracy: 95\nIteration: 2000. Loss: 0.0865858644247055. Accuracy: 96\nIteration: 2500. Loss: 0.07156120240688324. Accuracy: 96\nIteration: 3000. Loss: 0.04664849117398262. Accuracy: 97\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#other-adaptive-algorithms","title":"Other Adaptive Algorithms","text":"<ul> <li>Other adaptive algorithms (like Adam, adapting learning rates)<ul> <li>Adagrad</li> <li>Adadelta</li> <li>Adamax</li> <li>RMSProp</li> </ul> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithm-5-adagrad","title":"Optimization Algorithm 5: Adagrad","text":"<pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n# Set seed\ntorch.manual_seed(0)\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Non-linearity\n        self.relu = nn.ReLU()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.relu(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\n# learning_rate = 0.001\n\noptimizer = torch.optim.Adagrad(model.parameters())\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as Variable\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, 28*28)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 0.2757369875907898. Accuracy: 92\nIteration: 1000. Loss: 0.1992958039045334. Accuracy: 93\nIteration: 1500. Loss: 0.2227272093296051. Accuracy: 94\nIteration: 2000. Loss: 0.18628711998462677. Accuracy: 94\nIteration: 2500. Loss: 0.1470586657524109. Accuracy: 95\nIteration: 3000. Loss: 0.11748368293046951. Accuracy: 95\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithm-6-adadelta","title":"Optimization Algorithm 6: Adadelta","text":"<pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n# Set seed\ntorch.manual_seed(0)\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Non-linearity\n        self.relu = nn.ReLU()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.relu(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\n# learning_rate = 0.001\n\noptimizer = torch.optim.Adadelta(model.parameters())\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as Variable\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = Variable(images.view(-1, 28*28))\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 0.26303035020828247. Accuracy: 93.95\nIteration: 1000. Loss: 0.08731874823570251. Accuracy: 95.83\nIteration: 1500. Loss: 0.11502093076705933. Accuracy: 96.87\nIteration: 2000. Loss: 0.03550947830080986. Accuracy: 97.12\nIteration: 2500. Loss: 0.042649827897548676. Accuracy: 97.54\nIteration: 3000. Loss: 0.03061559610068798. Accuracy: 97.45\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithm-6-adamax","title":"Optimization Algorithm 6: Adamax","text":"<pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n# Set seed\ntorch.manual_seed(0)\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Non-linearity\n        self.relu = nn.ReLU()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.relu(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\n# learning_rate = 0.001\n\noptimizer = torch.optim.Adamax(model.parameters())\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as Variable\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, 28*28)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 0.29930350184440613. Accuracy: 92\nIteration: 1000. Loss: 0.18749120831489563. Accuracy: 93\nIteration: 1500. Loss: 0.21887679398059845. Accuracy: 95\nIteration: 2000. Loss: 0.14390651881694794. Accuracy: 95\nIteration: 2500. Loss: 0.10771607607603073. Accuracy: 96\nIteration: 3000. Loss: 0.0839928686618805. Accuracy: 96\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#optimization-algorithm-7-rmsprop","title":"Optimization Algorithm 7: RMSProp","text":"<pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n# Set seed\ntorch.manual_seed(0)\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Non-linearity\n        self.relu = nn.ReLU()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.relu(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\n# learning_rate = 0.001\n\noptimizer = torch.optim.RMSprop(model.parameters())\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as Variable\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, 28*28).requires_grad_()\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 0.25550296902656555. Accuracy: 95\nIteration: 1000. Loss: 0.17357593774795532. Accuracy: 93\nIteration: 1500. Loss: 0.10597744584083557. Accuracy: 96\nIteration: 2000. Loss: 0.03807783126831055. Accuracy: 96\nIteration: 2500. Loss: 0.10654022544622421. Accuracy: 96\nIteration: 3000. Loss: 0.05745543912053108. Accuracy: 96\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#summary-of-optimization-algorithms-performance","title":"Summary of Optimization Algorithms Performance","text":"<ul> <li>SGD: 95.78%</li> <li>SGD Momentum: 97.69%</li> <li>SGD Nesterov: 97.58%</li> <li>Adam: 97.20%</li> <li>Adagrad: 95.51%</li> <li>Adadelta: 97.45%</li> <li>Adamax: 96.58%</li> <li>RMSProp: 97.1%</li> </ul> <p>Performance is not definitive here</p> <p>I have used a seed to ensure you can reproduce results here. However, if you change the seed number you would realize that the performance of these optimization algorithms would change. A solution is to run each optimization on many seeds and get the average performance. Then you can compare the mean performance across all optimization algorithms. </p> <p>There are a lot of other factors like how Adam and SGD Momentum may have different ideal starting learning rates and require different learning rate scheduling. But off the hand, SGD and Adam are very robust optimization algorithms that you can rely on. </p> <p>Subsequently, we will look into more advanced optimization algorithms that are based mainly on SGD and Adam.</p>"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#simple-suggestions","title":"Simple Suggestions","text":"<ul> <li>Momentum/Nesterov<ul> <li>Powerful if we control the learning rate schedule</li> </ul> </li> <li>Adam<ul> <li>Lazy to control the learning rate schedule</li> </ul> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#summary","title":"Summary","text":"<p>We've learnt...</p> <p>Success</p> <ul> <li> Recap of 7 step process<ul> <li> Step 1: Load Dataset</li> <li> Step 2: Make Dataset Iterable</li> <li> Step 3: Create Model Class</li> <li> Step 4: Instantiate Model Class</li> <li> Step 5: Instantiate Loss Class</li> <li> Step 6: Instantiate Optimizer Class</li> <li> Step 7: Train Model</li> </ul> </li> <li> Step 6<ul> <li> Update parameters using gradients</li> <li> <code>parameters = parameters - learning_rate * parameters_gradients</code></li> </ul> </li> <li> Gradient descent<ul> <li> Using gradients (error signals from loss class) to update parameters</li> </ul> </li> <li> Mathematical interpretation: \\(\\theta = \\theta - \\eta \\cdot  \\nabla J(\\theta)\\)</li> <li> Optimisation Algorithms<ul> <li> Batch gradient descent</li> <li> Stochastic gradient descent</li> <li> Mini-batch gradient descent (SGD)</li> <li> SGD + Momentum</li> <li> SGD + Nesterov</li> <li> Adam</li> <li> Other adaptive algorithms: adagrad, adamax, adadelta, RMSProp</li> </ul> </li> <li> Recommendations<ul> <li> SGD+M</li> <li> SGD+N</li> <li> Adam</li> </ul> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/optimizers/#citation","title":"Citation","text":"<p>If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.</p> <p> </p>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/","title":"Weight Initializations &amp; Activation Functions","text":"<p>Run Jupyter Notebook</p> <p>You can run the code for this section in this jupyter notebook link.</p>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#recap-of-logistic-regression","title":"Recap of Logistic Regression","text":""},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#recap-of-feedforward-neural-network-activation-function","title":"Recap of Feedforward Neural Network Activation Function","text":""},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#sigmoid-logistic","title":"Sigmoid (Logistic)","text":"<ul> <li>\\(\\sigma(x) = \\frac{1}{1 + e^{-x}}\\)</li> <li>Input number \\(\\rightarrow\\) [0, 1]<ul> <li>Large negative number \\(\\rightarrow\\) 0</li> <li>Large positive number \\(\\rightarrow\\) 1</li> </ul> </li> <li>Cons: <ol> <li>Activation saturates at 0 or 1 with gradients \\(\\approx\\) 0<ul> <li>No signal to update weights \\(\\rightarrow\\) cannot learn</li> <li>Solution: Have to carefully initialize weights to prevent this</li> </ul> </li> </ol> </li> </ul> <pre><code>import matplotlib.pyplot as plt\n%matplotlib inline\nimport numpy as np\n\ndef sigmoid(x):\n    a = []\n    for item in x:\n        a.append(1/(1+np.exp(-item)))\n    return a\n\nx = np.arange(-10., 10., 0.2)\nsig = sigmoid(x)\n\nplt.style.use('ggplot')\nplt.plot(x,sig, linewidth=3.0)\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#tanh","title":"Tanh","text":"<ul> <li>\\(\\tanh(x) = 2 \\sigma(2x) -1\\)<ul> <li>A scaled sigmoid function</li> </ul> </li> <li>Input number \\(\\rightarrow\\) [-1, 1]</li> <li>Cons: <ol> <li>Activation saturates at 0 or 1 with gradients \\(\\approx\\) 0<ul> <li>No signal to update weights \\(\\rightarrow\\) cannot learn</li> <li>Solution: Have to carefully initialize weights to prevent this</li> </ul> </li> </ol> </li> </ul> <pre><code>x = np.arange(-10., 10., 0.2)\ntanh = np.dot(2, sigmoid(np.dot(2, x))) - 1\n\nplt.plot(x,tanh, linewidth=3.0)\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#relus","title":"ReLUs","text":"<ul> <li>\\(f(x) = \\max(0, x)\\)</li> <li>Pros:<ol> <li>Accelerates convergence \\(\\rightarrow\\) train faster</li> <li>Less computationally expensive operation compared to Sigmoid/Tanh exponentials</li> </ol> </li> <li>Cons:<ol> <li>Many ReLU units \"die\" \\(\\rightarrow\\) gradients = 0 forever<ul> <li>Solution: careful learning rate and weight initialization choice</li> </ul> </li> </ol> </li> </ul> <pre><code>x = np.arange(-10., 10., 0.2)\nrelu = np.maximum(x, 0)\n\nplt.plot(x,relu, linewidth=3.0)\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#why-do-we-need-weight-initializations-or-new-activation-functions","title":"Why do we need weight initializations or new activation functions?","text":"<ul> <li>To prevent vanishing/exploding gradients</li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#case-1-sigmoidtanh","title":"Case 1: Sigmoid/Tanh","text":"<ul> <li>Problem<ul> <li>If variance of input too large: gradients = 0 (vanishing gradients)</li> <li>If variance of input too small: linear \\(\\rightarrow\\) gradients = constant value</li> </ul> </li> <li>Solutions<ul> <li>Want a constant variance of input to achieve non-linearity \\(\\rightarrow\\) unique gradients for unique updates<ul> <li>Xavier Initialization (good constant variance for Sigmoid/Tanh)</li> <li>ReLU or Leaky ReLU</li> </ul> </li> </ul> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#case-2-relu","title":"Case 2: ReLU","text":"<ul> <li>Solution to Case 1<ul> <li>Regardless of variance of input: gradients = 0 or 1 </li> </ul> </li> <li>Problem<ul> <li>But those with 0: no updates (\"dead ReLU units\") </li> <li>Has unlimited output size with input &gt; 0 (explodes gradients subsequently)</li> </ul> </li> <li>Solutions<ul> <li>He Initialization (good constant variance)</li> <li>Leaky ReLU</li> </ul> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#case-3-leaky-relu","title":"Case 3: Leaky ReLU","text":"<ul> <li>Solution to Case 2<ul> <li>Solves the 0 signal issue when input &lt; 0  </li> </ul> </li> <li>Problem<ul> <li>Has unlimited output size with input &gt; 0 (explodes)</li> </ul> </li> <li>Solution<ul> <li>He Initialization (good constant variance)</li> </ul> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#summary-of-weight-initialization-solutions-to-activations","title":"Summary of weight initialization solutions to activations","text":"<ul> <li>Tanh/Sigmoid vanishing gradients can be solved with Xavier initialization<ul> <li>Good range of constant variance</li> </ul> </li> <li>ReLU/Leaky ReLU exploding gradients can be solved with He initialization<ul> <li>Good range of constant variance</li> </ul> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#types-of-weight-intializations","title":"Types of weight intializations","text":""},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#zero-initialization-set-all-weights-to-0","title":"Zero Initialization: set all weights to 0","text":"<ul> <li>Every neuron in the network computes the same output \\(\\rightarrow\\) computes the same gradient \\(\\rightarrow\\) same parameter updates </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#normal-initialization-set-all-weights-to-random-small-numbers","title":"Normal Initialization: set all weights to random small numbers","text":"<ul> <li>Every neuron in the network computes different output \\(\\rightarrow\\) computes different gradient \\(\\rightarrow\\) different parameter updates </li> <li>\"Symmetry breaking\" </li> <li>Problem: variance that grows with the number of inputs</li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#lecun-initialization-normalize-variance","title":"Lecun Initialization: normalize variance","text":"<ul> <li>Solves growing variance with the number of inputs \\(\\rightarrow\\) constant variance </li> <li>Look at a simple feedforward neural network </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#equations-for-lecun-initialization","title":"Equations for Lecun Initialization","text":"<ul> <li>\\(Y = AX + B\\)</li> <li>\\(y = a_1x_1 + a_2x_2 + \\cdot + a_n x_n + b\\)</li> <li>\\(Var(y) = Var(a_1x_1 + a_2x_2 + \\cdot + a_n x_n + b)\\)</li> <li>\\(Var(a_i x_i) = E(x_i)^2 Var(a_i) + E(a_i)^2Var(x_i) + Var(a_i)Var(x_i)\\)<ul> <li>General term, you might be more familiar with the following<ul> <li>\\(Var(XY) = E(X)^2 Var(Y) + E(Y)^2Var(X) + Var(X)Var(Y)\\)</li> </ul> </li> <li>\\(E(x_i)\\): expectation/mean of \\(x_i\\)</li> <li>\\(E(a_i)\\): expectation/mean of \\(a_i\\)</li> </ul> </li> <li>Assuming inputs/weights drawn i.i.d. with Gaussian distribution of mean=0<ul> <li>\\(E(x_i) = E(a_i) = 0\\)</li> <li>\\(Var(a_i x_i) = Var(a_i)Var(x_i)\\)</li> </ul> </li> <li>\\(Var(y) = Var(a_1)Var(x_1) + \\cdot + Var(a_n)Var(x_n)\\)<ul> <li>Since the bias, b, is a constant, \\(Var(b) = 0\\)</li> </ul> </li> <li>Since i.i.d.<ul> <li>\\(Var(y) = n \\times Var(a_i)Var(x_i)\\)</li> </ul> </li> <li>Since we want constant variance where \\(Var(y) = Var(x_i)\\)<ul> <li>\\(1 = nVar(a_i)\\)</li> <li>\\(Var(a_i) = \\frac{1}{n}\\)</li> </ul> </li> <li>This is essentially Lecun initialization, from his paper titled \"Efficient Backpropagation\"<ul> <li>We draw our weights i.i.d. with mean=0 and variance = \\(\\frac{1}{n}\\)</li> <li>Where \\(n\\) is the number of input units in the weight tensor</li> </ul> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#improvements-to-lecun-intialization","title":"Improvements to Lecun Intialization","text":"<ul> <li>They are essentially slight modifications to Lecun'98 initialization</li> <li>Xavier Intialization<ul> <li>Works better for layers with Sigmoid activations </li> <li>\\(var(a_i) = \\frac{1}{n_{in} + n_{out}}\\)<ul> <li>Where \\(n_{in}\\) and \\(n_{out}\\) are the number of input and output units in the weight tensor respectively</li> </ul> </li> </ul> </li> <li>Kaiming Initialization<ul> <li>Works better for layers with ReLU or LeakyReLU activations </li> <li>\\(var(a_i) = \\frac{2}{n_{in}}\\)</li> </ul> </li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#summary-of-weight-initializations","title":"Summary of  weight initializations","text":"<ul> <li>Normal Distribution</li> <li>Lecun Normal Distribution</li> <li>Xavier (Glorot) Normal Distribution </li> <li>Kaiming (He) Normal Distribution</li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#weight-initializations-with-pytorch","title":"Weight Initializations with PyTorch","text":""},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#normal-initialization-tanh-activation","title":"Normal Initialization: Tanh Activation","text":"<pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\nfrom torch.autograd import Variable\n\n# Set seed\ntorch.manual_seed(0)\n\n# Scheduler import\nfrom torch.optim.lr_scheduler import StepLR\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Linear weight, W,  Y = WX + B\n        nn.init.normal_(self.fc1.weight, mean=0, std=1)\n        # Non-linearity\n        self.tanh = nn.Tanh()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n        nn.init.normal_(self.fc2.weight, mean=0, std=1)\n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.tanh(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)\n\n'''\nSTEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS\n'''\n# step_size: at how many multiples of epoch you decay\n# step_size = 1, after every 2 epoch, new_lr = lr*gamma \n# step_size = 2, after every 2 epoch, new_lr = lr*gamma \n\n# gamma = decaying factor\nscheduler = StepLR(optimizer, step_size=1, gamma=0.96)\n\n'''\nSTEP 8: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    # Decay Learning Rate\n    scheduler.step()\n    # Print Learning Rate\n    print('Epoch:', epoch,'LR:', scheduler.get_lr())\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as tensors with gradient accumulation abilities\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, 28*28)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n               # Total correct predictions\n                correct += (predicted.type(torch.FloatTensor).cpu() == labels.type(torch.FloatTensor)).sum()\n\n            accuracy = 100. * correct.item() / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Epoch: 0 LR: [0.1]\nIteration: 500. Loss: 0.5192779302597046. Accuracy: 87.9\nEpoch: 1 LR: [0.096]\nIteration: 1000. Loss: 0.4060308337211609. Accuracy: 90.15\nEpoch: 2 LR: [0.09216]\nIteration: 1500. Loss: 0.2880493104457855. Accuracy: 90.71\nEpoch: 3 LR: [0.08847359999999999]\nIteration: 2000. Loss: 0.23173095285892487. Accuracy: 91.99\nEpoch: 4 LR: [0.084934656]\nIteration: 2500. Loss: 0.23814399540424347. Accuracy: 92.32\nIteration: 3000. Loss: 0.19513173401355743. Accuracy: 92.55\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#lecun-initialization-tanh-activation","title":"Lecun Initialization: Tanh Activation","text":"<ul> <li>By default, PyTorch uses Lecun initialization, so nothing new has to be done here compared to using Normal, Xavier or Kaiming initialization.</li> </ul> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\nfrom torch.autograd import Variable\n\n# Set seed\ntorch.manual_seed(0)\n\n# Scheduler import\nfrom torch.optim.lr_scheduler import StepLR\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Non-linearity\n        self.tanh = nn.Tanh()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.tanh(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)\n\n'''\nSTEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS\n'''\n# step_size: at how many multiples of epoch you decay\n# step_size = 1, after every 2 epoch, new_lr = lr*gamma \n# step_size = 2, after every 2 epoch, new_lr = lr*gamma \n\n# gamma = decaying factor\nscheduler = StepLR(optimizer, step_size=1, gamma=0.96)\n\n'''\nSTEP 8: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    # Decay Learning Rate\n    scheduler.step()\n    # Print Learning Rate\n    print('Epoch:', epoch,'LR:', scheduler.get_lr())\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as tensors with gradient accumulation abilities\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, 28*28)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted.type(torch.FloatTensor).cpu() == labels.type(torch.FloatTensor)).sum()\n\n            accuracy = 100. * correct.item() / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Epoch: 0 LR: [0.1]\nIteration: 500. Loss: 0.20123475790023804. Accuracy: 95.63\nEpoch: 1 LR: [0.096]\nIteration: 1000. Loss: 0.10885068774223328. Accuracy: 96.48\nEpoch: 2 LR: [0.09216]\nIteration: 1500. Loss: 0.1296212077140808. Accuracy: 97.22\nEpoch: 3 LR: [0.08847359999999999]\nIteration: 2000. Loss: 0.05178885534405708. Accuracy: 97.36\nEpoch: 4 LR: [0.084934656]\nIteration: 2500. Loss: 0.02619408629834652. Accuracy: 97.61\nIteration: 3000. Loss: 0.02096685953438282. Accuracy: 97.7\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#xavier-initialization-tanh-activation","title":"Xavier Initialization: Tanh Activation","text":"<pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\nfrom torch.autograd import Variable\n\n# Set seed\ntorch.manual_seed(0)\n\n# Scheduler import\nfrom torch.optim.lr_scheduler import StepLR\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Linear weight, W,  Y = WX + B\n        nn.init.xavier_normal_(self.fc1.weight)\n        # Non-linearity\n        self.tanh = nn.Tanh()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n        nn.init.xavier_normal_(self.fc2.weight)\n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.tanh(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)\n\n'''\nSTEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS\n'''\n# step_size: at how many multiples of epoch you decay\n# step_size = 1, after every 2 epoch, new_lr = lr*gamma \n# step_size = 2, after every 2 epoch, new_lr = lr*gamma \n\n# gamma = decaying factor\nscheduler = StepLR(optimizer, step_size=1, gamma=0.96)\n\n'''\nSTEP 8: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    # Decay Learning Rate\n    scheduler.step()\n    # Print Learning Rate\n    print('Epoch:', epoch,'LR:', scheduler.get_lr())\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as tensors with gradient accumulation abilities\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, 28*28)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted.type(torch.FloatTensor).cpu() == labels.type(torch.FloatTensor)).sum()\n\n            accuracy = 100. * correct.item() / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Epoch: 0 LR: [0.1]\nIteration: 500. Loss: 0.14800140261650085. Accuracy: 95.43\nEpoch: 1 LR: [0.096]\nIteration: 1000. Loss: 0.17138008773326874. Accuracy: 96.58\nEpoch: 2 LR: [0.09216]\nIteration: 1500. Loss: 0.07987994700670242. Accuracy: 96.95\nEpoch: 3 LR: [0.08847359999999999]\nIteration: 2000. Loss: 0.07756654918193817. Accuracy: 97.23\nEpoch: 4 LR: [0.084934656]\nIteration: 2500. Loss: 0.05563584715127945. Accuracy: 97.6\nIteration: 3000. Loss: 0.07122127711772919. Accuracy: 97.49\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#xavier-initialization-relu-activation","title":"Xavier Initialization: ReLU Activation","text":"<pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\nfrom torch.autograd import Variable\n\n# Set seed\ntorch.manual_seed(0)\n\n# Scheduler import\nfrom torch.optim.lr_scheduler import StepLR\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Linear weight, W,  Y = WX + B\n        nn.init.xavier_normal_(self.fc1.weight)\n        # Non-linearity\n        self.relu = nn.ReLU()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n        nn.init.xavier_normal_(self.fc2.weight)\n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.relu(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)\n\n'''\nSTEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS\n'''\n# step_size: at how many multiples of epoch you decay\n# step_size = 1, after every 2 epoch, new_lr = lr*gamma \n# step_size = 2, after every 2 epoch, new_lr = lr*gamma \n\n# gamma = decaying factor\nscheduler = StepLR(optimizer, step_size=1, gamma=0.96)\n\n'''\nSTEP 8: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    # Decay Learning Rate\n    scheduler.step()\n    # Print Learning Rate\n    print('Epoch:', epoch,'LR:', scheduler.get_lr())\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as tensors with gradient accumulation abilities\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, 28*28)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted.type(torch.FloatTensor).cpu() == labels.type(torch.FloatTensor)).sum()\n\n            accuracy = 100. * correct.item() / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Epoch: 0 LR: [0.1]\nIteration: 500. Loss: 0.1245984435081482. Accuracy: 95.82\nEpoch: 1 LR: [0.096]\nIteration: 1000. Loss: 0.14348150789737701. Accuracy: 96.72\nEpoch: 2 LR: [0.09216]\nIteration: 1500. Loss: 0.10421314090490341. Accuracy: 97.3\nEpoch: 3 LR: [0.08847359999999999]\nIteration: 2000. Loss: 0.04693891853094101. Accuracy: 97.29\nEpoch: 4 LR: [0.084934656]\nIteration: 2500. Loss: 0.06869587302207947. Accuracy: 97.61\nIteration: 3000. Loss: 0.056865859776735306. Accuracy: 97.48\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#he-initialization-relu-activation","title":"He Initialization: ReLU Activation","text":"<pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\nfrom torch.autograd import Variable\n\n# Set seed\ntorch.manual_seed(0)\n\n# Scheduler import\nfrom torch.optim.lr_scheduler import StepLR\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Linear weight, W,  Y = WX + B\n        nn.init.kaiming_normal_(self.fc1.weight)\n        # Non-linearity\n        self.relu = nn.ReLU()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n        nn.init.kaiming_normal_(self.fc2.weight)\n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.relu(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)\n\n'''\nSTEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS\n'''\n# step_size: at how many multiples of epoch you decay\n# step_size = 1, after every 2 epoch, new_lr = lr*gamma \n# step_size = 2, after every 2 epoch, new_lr = lr*gamma \n\n# gamma = decaying factor\nscheduler = StepLR(optimizer, step_size=1, gamma=0.96)\n\n'''\nSTEP 8: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    # Decay Learning Rate\n    scheduler.step()\n    # Print Learning Rate\n    print('Epoch:', epoch,'LR:', scheduler.get_lr())\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as tensors with gradient accumulation abilities\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, 28*28)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted.type(torch.FloatTensor).cpu() == labels.type(torch.FloatTensor)).sum()\n\n            accuracy = 100. * correct.item() / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Epoch: 0 LR: [0.1]\nIteration: 500. Loss: 0.11658752709627151. Accuracy: 95.7\nEpoch: 1 LR: [0.096]\nIteration: 1000. Loss: 0.15525035560131073. Accuracy: 96.65\nEpoch: 2 LR: [0.09216]\nIteration: 1500. Loss: 0.09970294684171677. Accuracy: 97.07\nEpoch: 3 LR: [0.08847359999999999]\nIteration: 2000. Loss: 0.04063304886221886. Accuracy: 97.23\nEpoch: 4 LR: [0.084934656]\nIteration: 2500. Loss: 0.0719323456287384. Accuracy: 97.7\nIteration: 3000. Loss: 0.04470040276646614. Accuracy: 97.39\n</code></pre>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#initialization-performance","title":"Initialization Performance","text":"Initialization: Activation Test Accuracy Normal: Tanh 92.55 Lecun: Tanh 97.7 Xavier: Tanh 97.49 Xavier: ReLU 97.48 He: ReLU 97.39 <p>Interpreting the Validation Accuracy Table</p> <p>Take note that these numbers would fluctuate slightly when you change seeds. </p> <p>However, the key point here is that all the other intializations are clearly much better than a basic normal distribution.</p> <p>Whether He, Xavier, or Lecun  intialization is better or any other initializations depends on the overall model's architecture (RNN/LSTM/CNN/FNN etc.), activation functions (ReLU, Sigmoid, Tanh etc.) and more.</p> <p>For example, more advanced initializations we will cover subsequently is orthogonal initialization that works better for RNN/LSTM. But due to the math involved in that, we will be covering such advanced initializations in a separate section.</p>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#summary","title":"Summary","text":"<p>We've learnt...</p> <p>Success</p> <ul> <li> Recap of LG</li> <li> Recap of FNN</li> <li> Recap of Activation Functions<ul> <li> Sigmoid (Logistic)</li> <li> Tanh</li> <li> ReLU</li> </ul> </li> <li> Need for Weight Initializations<ul> <li> Sigmoid/Tanh: vanishing gradients<ul> <li> Constant Variance initialization with Lecun or Xavier </li> </ul> </li> <li> ReLU: exploding gradients with dead units<ul> <li> He Initialization</li> </ul> </li> <li> Leaky ReLU: exploding gradients only<ul> <li> He Initialization</li> </ul> </li> </ul> </li> <li> Types of weight initialisations<ul> <li> Zero</li> <li> Normal: growing weight variance</li> <li> Lecun: constant variance</li> <li> Xavier: constant variance for Sigmoid/Tanh</li> <li> Kaiming He: constant variance for ReLU activations</li> </ul> </li> <li> PyTorch implementation</li> </ul>"},{"location":"deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/#citation","title":"Citation","text":"<p>If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.</p> <p> </p>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/","title":"Markov Decision Processes (MDP) and Bellman Equations","text":""},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#markov-decision-processes-mdps","title":"Markov Decision Processes (MDPs)","text":"<ul> <li>Typically we can frame all RL tasks as MDPs<sup>1</sup><ul> <li>Intuitively, it's sort of a way to frame RL tasks such that we can solve them in a \"principled\" manner. We will go into the specifics throughout this tutorial</li> </ul> </li> <li>The key in MDPs is the Markov Property<ul> <li>Essentially the future depends on the present and not the past<ul> <li>More specifically, the future is independent of the past given the present</li> <li>There's an assumption the present state encapsulates past information. This is not always true, see the note below.</li> </ul> </li> </ul> </li> <li>Putting into the context of what we have covered so far: our agent can (1) control its action based on its current (2) completely known state<ul> <li>Back to the \"driving to avoid puppy\" example: given we know there is a dog in front of the car as the current state and the car is always moving forward (no reverse driving), the agent can decide to take a left/right turn to avoid colliding with the puppy in front</li> </ul> </li> </ul>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#two-main-characteristics-for-mdps","title":"Two main characteristics for MDPs","text":"<ol> <li>Control over state transitions</li> <li>States completely observable</li> </ol> <p>Other Markov Models</p> <p>Permutations of whether there is presence of the two main characteristics would lead to different Markov models. These are not important now, but it gives you an idea of what other frameworks we can use besides MDPs.</p>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#types-of-markov-models","title":"Types of Markov Models","text":"<ol> <li>{++Control++} over state transitions and {++completely observable++} states: MDPs</li> <li>{++Control++} over state transitions and partially observable states: Partially Observable MDPs (POMDPs)</li> <li>No control over state transitions and {++completely observable++} states: Markov Chain</li> <li>No control over state transitions and partially observable states: Hidden Markov Model</li> </ol>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#pomdps","title":"POMDPs","text":"<ul> <li>Imagine our driving example where we don't know if the car is going forward/backward in its state, but only know there is a puppy in the center lane in front, this is a partially observable state</li> <li>There are ways to counter this<ol> <li>Use the complete history to construct the current state</li> <li>Represent the current state as a probability distribution (bayesian approach) of what the agent perceives of the current state</li> <li>Using a RNN to form the current state that encapsulates the past<sup>2</sup></li> </ol> </li> </ul>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#5-components-of-mdps","title":"5 Components of MDPs","text":"<ol> <li>\\(\\mathcal{S}\\): set of states</li> <li>\\(\\mathcal{A}\\): set of actions</li> <li>\\(\\mathcal{R}\\): reward function</li> <li>\\(\\mathcal{P}\\): transition probability function</li> <li>\\(\\gamma\\): discount for future rewards</li> </ol> <p>Remembering 5 components with a mnemonic</p> <p>A mnemonic I use to remember the 5 components is the acronym \"SARPY\" (sar-py). I know \\(\\gamma\\) is not \\(\\mathcal{Y}\\) but it looks like a <code>y</code> so there's that.</p>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#moving-from-mdps-to-optimal-policy","title":"Moving From MDPs to Optimal Policy","text":"<ul> <li>We have an agent acting in an environment</li> <li>The way the environment reacts to the agent's actions (\\(a\\)) is dictated by a model</li> <li>The agent can take actions (\\(a\\)) to move from one state (\\(s\\)) to another new state (\\(s')\\)</li> <li>When the agent has transited to a new state (\\(s')\\), there will a reward (\\(r\\))</li> <li>We may or may not know our model<ol> <li>Model-based RL: this is where we can {++clearly define++} our (1) transition probabilities and/or (2) reward function<ul> <li>A global minima can be attained via Dynamic Programming (DP)</li> </ul> </li> <li>Model-free RL: this is where we cannot clearly define our (1) transition probabilities and/or (2) reward function<ul> <li>Most real-world problems are under this category so we will mostly place our attention on this category</li> </ul> </li> </ol> </li> <li>How the agent acts (\\(a\\)) in its current state (\\(s\\)) is specified by its policy (\\(\\pi(s)\\))<ul> <li>It can either be deterministic or stochastic<ol> <li>Deterministic policy: \\(a = \\pi(s)\\)</li> <li>Stochastic policy: \\(\\mathbb{P}_\\pi [A=a \\vert S=s] = \\pi(a | s)\\)<ul> <li>This is the proability of taking an action given the current state under the policy</li> <li>\\(\\mathcal{A}\\): set of all actions</li> <li>\\(\\mathcal{S}\\): set of all states</li> </ul> </li> </ol> </li> </ul> </li> <li>When the agent acts given its state under the policy (\\(\\pi(a | s)\\)), the transition probability function \\(\\mathcal{P}\\) determines the subsequent state (\\(s'\\))<ul> <li>\\(\\mathcal{P}_{ss'}^a = \\mathcal{P}(s' \\vert s, a)  = \\mathbb{P} [S_{t+1} = s' \\vert S_t = s, A_t = a]\\)</li> </ul> </li> <li>When the agent act based on its policy (\\(\\pi(a | s)\\)) and transited to the new state determined by the transition probability function \\(\\mathcal{P}_{ss'}^a\\) it gets a reward based on the reward function as a feedback<ul> <li>\\(\\mathcal{R}_s^a = \\mathbb{E} [\\mathcal{R}_{t+1} \\vert S_t = s, A_t = a]\\)</li> </ul> </li> <li>Rewards are short-term, given as feedback after the agent takes an action and transits to a new state. Summing all future rewards and discounting them would lead to our return \\(\\mathcal{G}\\)<ul> <li>\\(\\mathcal{G}_t = \\sum_{i=0}^{N} \\gamma^k \\mathcal{R}_{t+1+i}\\)<ul> <li>\\(\\gamma\\), our discount factor which ranges from 0 to 1 (inclusive) reduces the weightage of future rewards allowing us to balance between short-term and long-term goals</li> </ul> </li> </ul> </li> <li>With our return \\(\\mathcal{G}\\), we then have our state-value function \\(\\mathcal{V}_{\\pi}\\) (how good to stay in that state) and our action-value or q-value function \\(\\mathcal{Q}_{\\pi}\\) (how good to take the action)<ul> <li>\\(\\mathcal{V}_{\\pi}(s) = \\mathbb{E}_{\\pi}[\\mathcal{G}_t \\vert \\mathcal{S}_t = s]\\)</li> <li>\\(\\mathcal{Q}_{\\pi}(s, a) = \\mathbb{E}_{\\pi}[\\mathcal{G}_t \\vert \\mathcal{S}_t = s, \\mathcal{A}_t = a]\\)</li> <li>The advantage function is simply the difference between the two functions \\(\\mathcal{A}_{\\pi}(s, a) = \\mathcal{Q}_{\\pi}(s, a) - \\mathcal{V}_{\\pi}(s)\\)<ul> <li>Seems useless at this stage, but this advantage function will be used in some key algorithms we are covering</li> </ul> </li> </ul> </li> <li>Since our policy determines how our agent acts given its state, achieving an optimal policy \\(\\pi_*\\) would mean achieving optimal actions that is exactly what we want!</li> </ul> <p>Basic Categories of Approaches</p> <p>We've covered state-value functions, action-value functions, model-free RL and model-based RL. They form general overarching categories of how we design our agent.</p>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#categories-of-design","title":"Categories of Design","text":"<ol> <li>State-value based: search for the optimal state-value function (goodness of action in the state)</li> <li>Action-value based: search for the optimal action-value function (goodness of policy)</li> <li>Actor-critic based: using both state-value and action-value function </li> <li>Model based: attempts to model the environment to find the best policy</li> <li>Model-free based: trial and error to optimize for the best policy to get the most rewards instead of modelling the environment explicitly</li> </ol>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#optimal-policy","title":"Optimal Policy","text":"<ul> <li>Optimal policy \\(\\pi_*\\) \u2192 optimal state-value and action-value functions \u2192 max return \u2192 argmax of value functions<ul> <li>\\(\\pi_{*} = \\arg\\max_{\\pi} \\mathcal{V}_{\\pi}(s) = \\arg\\max_{\\pi} \\mathcal{Q}_{\\pi}(s, a)\\)</li> </ul> </li> <li>To calculate argmax of value functions \u2192 we need max return \\(\\mathcal{G}_t\\) \u2192 need max sum of rewards \\(\\mathcal{R}_s^a\\)</li> <li>To get max sum of rewards \\(\\mathcal{R}_s^a\\) we will rely on the Bellman Equations.<sup>3</sup></li> </ul>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#bellman-equation","title":"Bellman Equation","text":"<ul> <li>Essentially, the Bellman Equation breaks down our value functions into two parts<ol> <li>Immediate reward</li> <li>Discounted future value function</li> </ol> </li> <li>State-value function can be broken into:<ul> <li>\\(\\begin{aligned}     \\mathcal{V}_{\\pi}(s) &amp;= \\mathbb{E}[\\mathcal{G}_t \\vert \\mathcal{S}_t = s] \\\\     &amp;= \\mathbb{E} [\\mathcal{R}_{t+1} + \\gamma \\mathcal{R}_{t+2} + \\gamma^2 \\mathcal{R}_{t+3} + \\dots \\vert \\mathcal{S}_t = s] \\\\     &amp;= \\mathbb{E} [\\mathcal{R}_{t+1} + \\gamma (\\mathcal{R}_{t+2} + \\gamma \\mathcal{R}_{t+3} + \\dots) \\vert \\mathcal{S}_t = s] \\\\     &amp;= \\mathbb{E} [\\mathcal{R}_{t+1} + \\gamma \\mathcal{G}_{t+1} \\vert \\mathcal{S}_t = s] \\\\     &amp;= \\mathbb{E} [\\mathcal{R}_{t+1} + \\gamma \\mathcal{V}_{\\pi}(\\mathcal{s}_{t+1}) \\vert \\mathcal{S}_t = s]     \\end{aligned}\\)</li> </ul> </li> <li>Action-value function can be broken into:<ul> <li>\\(\\mathcal{Q}_{\\pi}(s, a) = \\mathbb{E} [\\mathcal{R}_{t+1} + \\gamma \\mathcal{Q}_{\\pi}(\\mathcal{s}_{t+1}, \\mathcal{a}_{t+1}) \\vert \\mathcal{S}_t = s, \\mathcal{A} = a]\\)</li> </ul> </li> </ul>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#key-recap-on-value-functions","title":"Key Recap on Value Functions","text":"<ul> <li>\\(\\mathcal{V}_{\\pi}(s) = \\mathbb{E}_{\\pi}[\\mathcal{G}_t \\vert \\mathcal{S}_t = s]\\)<ul> <li>State-value function: tells us how good to be in that state</li> </ul> </li> <li>\\(\\mathcal{Q}_{\\pi}(s, a) = \\mathbb{E}_{\\pi}[\\mathcal{G}_t \\vert \\mathcal{S}_t = s, \\mathcal{A}_t = a]\\)<ul> <li>Action-value function: tells us how good to take actions given state</li> </ul> </li> </ul>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#bellman-expectation-equations","title":"Bellman Expectation Equations","text":"<ul> <li>Now we can move from Bellman Equations into Bellman Expectation Equations</li> <li>Basic: State-value function \\(\\mathcal{V}_{\\pi}(s)\\)<ol> <li>Current state \\(\\mathcal{S}\\)</li> <li>Multiple possible actions determined by stochastic policy \\(\\pi(a | s)\\)</li> <li>Each possible action is associated with a action-value function \\(\\mathcal{Q}_{\\pi}(s, a)\\) returning a value of that particular action</li> <li>Multiplying the possible actions with the action-value function and summing them gives us an indication of how good it is to be in that state<ul> <li>Final equation: \\(\\mathcal{V}_{\\pi}(s) = \\sum_{a \\in \\mathcal{A}} \\pi(a | s) \\mathcal{Q}(s, a)\\)</li> <li>Loose intuitive interpretation: <ul> <li>state-value = sum(policy determining actions * respective action-values)</li> </ul> </li> </ul> </li> </ol> </li> <li>Basic: Action-value function \\(\\mathcal{Q}_{\\pi}(s, a)\\)<ol> <li>With a list of possible multiple actions, there is a list of possible subsequent states \\(s'\\) associated with:<ol> <li>state value function \\(\\mathcal{V}_{\\pi}(s')\\) </li> <li>transition probability function \\(\\mathcal{P}_{ss'}^a\\) determining where the agent could land in based on the action</li> <li>reward \\(\\mathcal{R}_s^a\\) for taking the action</li> </ol> </li> <li>Summing the reward and the transition probability function associated with the state-value function gives us an indication of how good it is to take the actions given our state<ul> <li>Final equation: \\(\\mathcal{Q}_{\\pi}(s, a) = \\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a {V}_{\\pi}(s')\\)</li> <li>Loose intuitive interpretation: <ul> <li>action-value = reward + sum(transition outcomes determining states * respective state-values)</li> </ul> </li> </ul> </li> </ol> </li> <li>Expanded functions (substitution)<ul> <li>Substituting action-value function into the state-value function<ul> <li>\\(\\mathcal{V}_{\\pi}(s) = \\sum_{a \\in \\mathcal{A}} \\pi(a | s) (\\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a {V}_{\\pi}(s'))\\)</li> </ul> </li> <li>Substituting  state-value function into action-value function<ul> <li>\\(\\mathcal{Q}_{\\pi}(s, a) = \\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a \\sum_{a' \\in \\mathcal{A}} \\pi(a' | s') \\mathcal{Q}(s', a')\\)</li> </ul> </li> </ul> </li> </ul>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#bellman-optimality-equations","title":"Bellman Optimality Equations","text":"<ul> <li>Remember optimal policy \\(\\pi_*\\) \u2192 optimal state-value and action-value functions \u2192 argmax of value functions<ul> <li>\\(\\pi_{*} = \\arg\\max_{\\pi} \\mathcal{V}_{\\pi}(s) = \\arg\\max_{\\pi} \\mathcal{Q}_{\\pi}(s, a)\\)</li> </ul> </li> <li>Finally with Bellman Expectation Equations derived from Bellman Equations, we can derive the equations for the argmax of our value functions</li> <li>Optimal state-value function<ul> <li>\\(\\mathcal{V}_*(s) = \\arg\\max_{\\pi} \\mathcal{V}_{\\pi}(s)\\)</li> <li>Given \\(\\mathcal{V}_{\\pi}(s) = \\sum_{a \\in \\mathcal{A}} \\pi(a | s) (\\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a {V}_{\\pi}(s'))\\)</li> <li>We have \\(\\mathcal{V}_*(s) = \\max_{a \\in \\mathcal{A}} (\\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a {V}_{*}(s')))\\)</li> </ul> </li> <li>Optimal action-value function<ul> <li>\\(\\mathcal{Q}_*(s) = \\arg\\max_{\\pi} \\mathcal{Q}_{\\pi}(s)\\)</li> <li>Given \\(\\mathcal{Q}_{\\pi}(s, a) = \\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a \\sum_{a' \\in \\mathcal{A}} \\pi(a' | s') \\mathcal{Q}(s', a')\\)</li> <li>We have \\(\\mathcal{Q}_{*}(s, a) = \\mathcal{R}_s^a + \\gamma \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a max_{a' \\in \\mathcal{A}} \\mathcal{Q}_{*}(s', a')\\)</li> </ul> </li> </ul>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/#optimal-action-value-and-state-value-functions","title":"Optimal Action-value and State-value functions","text":"<ol> <li>If the entire environment is known, such that we know our reward function and transition probability function, then we can solve for the optimal action-value and state-value functions via Dynamic Programming like<ul> <li>Policy evaluation, policy improvement, and policy iteration</li> </ul> </li> <li>However, typically we don't know the environment entirely then there is not closed form solution in getting optimal action-value and state-value functions. Hence, we need other iterative approaches like<ol> <li>Monte-Carlo methods</li> <li>Temporal difference learning (model-free and learns with episodes)<ol> <li>On-policy TD: SARSA</li> <li>Off-policy TD: Q-Learning and Deep Q-Learning (DQN)</li> </ol> </li> <li>Policy gradient<ul> <li>REINFORCE</li> <li>Actor-Critic</li> <li>A2C/A3C</li> <li>ACKTR</li> <li>PPO</li> <li>DPG</li> <li>DDPG (DQN + DPG)</li> </ul> </li> </ol> </li> </ol> <p>Closed form solution</p> <p>If there is a closed form solution, then the variables' values can be obtained with a finite number of mathematical operations (for example add, subtract, divide, and multiply).</p> <p>For example, solving \\(2x = 8 - 6x\\) would yield \\(8x = 8\\) by adding \\(6x\\) on both sides of the equation and finally yielding the value of \\(x=1\\) by dividing both sides of the equation by \\(8\\). </p> <p>These finite 2 steps of mathematical operations allowed us to solve for the value of x as the equation has a closed-form solution.</p> <p>However, many cases in deep learning and reinforcement learning there are no closed-form solutions which requires all the iterative methods mentioned above.</p> <ol> <li> <p>Bellman, R. A Markovian Decision Process. Journal of Mathematics and Mechanics. 1957.\u00a0\u21a9</p> </li> <li> <p>Matthew J. Hausknecht and Peter Stone. Deep Recurrent Q-Learning for Partially Observable MDPs. 2015.\u00a0\u21a9</p> </li> <li> <p>R Bellman. On the Theory of Dynamic Programming. Proceedings of the National Academy of Sciences. 1952.\u00a0\u21a9</p> </li> </ol>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/","title":"Dynamic Programming","text":""},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#frozen-lake-introduction","title":"Frozen Lake Introduction","text":"<p>Run Jupyter Notebook</p> <p>You can run the code for this section in this jupyter notebook link.</p> <ul> <li>Fronze Lake is a simple game where you are on a frozen lake and you need to retrieve an item on the frozen lake where some parts are frozen and some parts are holes (if you walk into them you die)</li> <li>Actions: \\(\\mathcal{A} = \\{0, 1, 2, 3\\}\\)<ol> <li>LEFT: 0</li> <li>DOWN = 1</li> <li>RIGHT = 2</li> <li>UP = 3</li> </ol> </li> <li>Whole lake is a 4 x 4 grid world, \\(\\mathcal{S} = \\{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\\}\\)<ul> <li></li> </ul> </li> <li>On each grid, there are 4 possibilities<ul> <li>S: starting point, safe (code = 'SFFF')</li> <li>F: frozen surface, safe (code = 'FHFH')</li> <li>H: hole, fall to your doom (code = 'FFFH')</li> <li>G: goal, where the frisbee is located ('HFFG')</li> <li></li> </ul> </li> </ul>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#goal-of-frozen-lake","title":"Goal of Frozen Lake","text":"<p>The key here is we want to get to G without falling into the hole H in the shortest amount of time</p>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#why-dynamic-programming","title":"Why Dynamic Programming?","text":"<p>In this game, we know our transition probability function and reward function, essentially the whole environment, allowing us to turn this game into a simple planning problem via dynamic programming through 4 simple functions: (1) policy evaluation (2) policy improvement (3) policy iteration or (4) value iteration</p> <p>Before we explore how to solve this game, let's first understand how the game works in detail.</p>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#deterministic-policy-environment","title":"Deterministic Policy Environment","text":"<p>Make OpenAI Gym Environment for Frozen Lake</p> <pre><code># Import gym, installable via `pip install gym`\nimport gym\n\n# Environment environment Slippery (stochastic policy, move left probability = 1/3) comes by default!\n# If we want deterministic policy, we need to create new environment\n# Make environment No Slippery (deterministic policy, move left = left)\n\ngym.envs.register(\n    id='FrozenLakeNotSlippery-v0',\n    entry_point='gym.envs.toy_text:FrozenLakeEnv',\n    kwargs={'map_name' : '4x4', 'is_slippery': False},\n    max_episode_steps=100,\n    reward_threshold=0.78, # optimum = .8196\n)\n\n# You can only register once\n# To delete any new environment\n# del gym.envs.registry.env_specs['FrozenLakeNotSlippery-v0']\n\n# Make the environment based on deterministic policy\nenv = gym.make('FrozenLakeNotSlippery-v0')\n</code></pre> <p>Observation space</p> <pre><code># State space\nprint(env.observation_space)\n</code></pre> <pre><code>Discrete(16)\n</code></pre> <p>State space</p> <pre><code>S_n = env.observation_space.n\nprint(S_n)\n</code></pre> <pre><code>16\n</code></pre> <p>Sampling state space</p> <pre><code># We should expect to see 15 possible grids from 0 to 15 when\n# we uniformly randomly sample from our observation space\nfor i in range(10):\n    print(env.observation_space.sample())\n</code></pre> <pre><code>11\n12\n7\n13\n7\n11\n14\n14\n4\n12\n</code></pre> <p>Action space</p> <pre><code># Action space\nprint(env.action_space)\n</code></pre> <pre><code>A_n = env.action_space.n\nprint(A_n)\n</code></pre> <pre><code>Discrete(4)\n\n4\n</code></pre> <p>Random sampling of actions</p> <pre><code># We should expect to see 4 actions when\n# we uniformly randomly sample:\n#     1. LEFT: 0\n#     2. DOWN = 1\n#     3. RIGHT = 2\n#     4. UP = 3\nfor i in range(10):\n    print(env.action_space.sample())\n</code></pre> <pre><code>0\n3\n1\n3\n3\n2\n3\n2\n2\n2\n</code></pre>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#making-steps","title":"Making Steps","text":"<p>Initial state</p> <pre><code># This sets the initial state at S, our starting point\n# We can render the environment to see where we are on the 4x4 frozenlake gridworld\nenv.reset()\nenv.render()\n</code></pre> <pre><code>[S]FFF\nFHFH\nFFFH\nHFFG\n</code></pre> <p>Go left</p> <pre><code># Go left (action=0), nothing should happen, and we should stay at the starting point, because there's no grid on the left\nenv.reset()\naction = 0\n(observation, reward, done, prob) = env.step(action)\nenv.render()\n\n# Observation = 1: move to grid number 1 (unchanged)\n# Prob = 1: deterministic policy, if we choose to go left, we'll go left\nprint(observation, reward, done, prob)\n</code></pre> <pre><code>(Left)\n[S]FFF\nFHFH\nFFFH\nHFFG\n\n0 0.0 False {'prob': 1.0}\n</code></pre> <p>Go down</p> <pre><code># Go down (action = 1), we should be safe as we step on frozen grid\nenv.reset()\naction = 1\n(observation, reward, done, prob) = env.step(action)\nenv.render()\n\n# Observation = 4: move to grid number 4\n# Prob = 1: deterministic policy, if we choose to go down we'll go down\nprint(observation, reward, done, prob)\n</code></pre> <pre><code>(Down)\nSFFF\n[F]HFH\nFFFH\nHFFG\n\n4 0.0 False {'prob': 1.0}\n</code></pre> <p>Go right</p> <pre><code># Go right (action = 2), we should be safe as we step on frozen grid\nenv.reset()\naction = 2\n(observation, reward, done, prob) = env.step(action)\nenv.render()\n\n# Observation = 1: move to grid number 1\n# Prob = 1: deterministic policy, if we choose to go right we'll go right\nprint(observation, reward, done, prob)\n</code></pre> <pre><code>(Right)\nS[F]FF\nFHFH\nFFFH\nHFFG\n\n1 0.0 False {'prob': 1.0}\n</code></pre> <p>Go right twice</p> <pre><code># Go right twice (action = 2), we should be safe as we step on 2 frozen grids\nenv.reset()\naction = 2\n(observation, reward, done, prob) = env.step(action)\nenv.render()\n(observation, reward, done, prob) = env.step(action)\nenv.render()\n\n# Observation = 2: move to the right twice from grid 0 to grid 2\n# Prob = 1: deterministic policy, if we choose to go right twice we'll go right twice\nprint(observation, reward, done, prob)\n</code></pre> <pre><code>(Right)\nS[F]FF\nFHFH\nFFFH\nHFFG\n\n(Right)\nSF[F]F\nFHFH\nFFFH\nHFFG\n\n2 0.0 False {'prob': 1.0}\n</code></pre>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#dying-drop-in-hole-grid-12-h","title":"Dying: drop in hole grid 12,  H","text":"<p>Go down thrice</p> <pre><code># Go down thrice (action = 1), we will die as we step onto the grid with a hole\nenv.reset()\naction = 1\n(observation, reward, done, prob) = env.step(action)\nenv.render()\n(observation, reward, done, prob) = env.step(action)\nenv.render()\n(observation, reward, done, prob) = env.step(action)\nenv.render()\n\n# Observation = 2: move to the right twice from grid 0 to grid 2\n# Prob = 1: deterministic policy, if we choose to go right twice we'll go right twice\n# Done = True because the game ends when we die (go onto hole grid (H) or finish the game (G))\nprint(observation, reward, done, prob)\n</code></pre> <pre><code>(Down)\nSFFF\n[F]HFH\nFFFH\nHFFG\n\n(Down)\nSFFF\nFHFH\n[F]FFH\nHFFG\n\n(Down)\nSFFF\nFHFH\nFFFH\n[H]FFG\n\n12 0.0 True {'prob': 1.0}\n</code></pre>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#winning-get-to-grid-15-g","title":"Winning: get to grid 15, G","text":"<p>Go right twice, go down thrice, go right once</p> <pre><code># Go right twice (action = 2), go down thrice (action = 1), go right once (action = 2)\nenv.reset()\n\n# Right Twice\naction = 2\n(observation, reward, done, prob) = env.step(action)\nenv.render()\n(observation, reward, done, prob) = env.step(action)\nenv.render()\n\n# Down Thrice\naction = 1\n(observation, reward, done, prob) = env.step(action)\nenv.render()\n(observation, reward, done, prob) = env.step(action)\nenv.render()\n(observation, reward, done, prob) = env.step(action)\nenv.render()\n\n# Right Once\naction = 2\n(observation, reward, done, prob) = env.step(action)\nenv.render()\n\n# Observation = 2: move to the right twice from grid 0 to grid 2\n# Prob = 1: deterministic policy, if we choose to go right twice we'll go right twice\n# Done = True because the game ends when we die (go onto hole grid (H) or finish the game (G))\nprint(observation, reward, done, prob)\n</code></pre> <pre><code>(Right)\nS[F]FF\nFHFH\nFFFH\nHFFG\n\n(Right)\nSF[F]F\nFHFH\nFFFH\nHFFG\n\n(Down)\nSFFF\nFH[F]H\nFFFH\nHFFG\n\n(Down)\nSFFF\nFHFH\nFF[F]H\nHFFG\n\n(Down)\nSFFF\nFHFH\nFFFH\nHF[F]mG\n\n(Right)\nSFFF\nFHFH\nFFFH\nHFF[G]\n\n15 1.0 True {'prob': 1.0}\n</code></pre>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#non-deterministic-policy-environment","title":"Non-deterministic Policy Environment","text":"<p>Go right?</p> <pre><code># Make the environment based on non-deterministic policy\nenv = gym.make('FrozenLake-v0')\n\n# Go right once (action = 2), we should go to the right but we did not!\nenv.seed(8)\nenv.reset()\naction = 2\n(observation, reward, done, prob) = env.step(action)\nenv.render()\n\n# Observation = 0: move to the right once from grid 0 to grid 1\n# Prob = 1/3: non-deterministic policy, if we choose to go right, there's only a 1/3 probability we would go to the right and with this environment seed we did not\nprint(observation, reward, done, prob)\n</code></pre> <pre><code>(Right)\n[S]FFF\nFHFH\nFFFH\nHFFG\n\n0 0.0 False {'prob': 0.3333333333333333}\n</code></pre> <p>Go right 10 times?</p> <pre><code># Try to go to the right 10 times, let's see how many times it goes to the right, by right we won't die because we would end up at the extreme right of grid 3\n# See how it can go down/left/up/nothing instead of just right? \n# Intuitively when we are moving on a frozen lake, some times when we want to walk one direction we may end up in another direction as it's slippery\n# Setting seed here of the environment so you can reproduce my results, otherwise stochastic policy will yield different results for each run\nenv.seed(8)\nenv.reset()\nfor i in range(10):\n    action = 2\n    (observation, reward, done, prob) = env.step(action)\n    env.render()\n</code></pre> <pre><code>(Right)\n[S]FFF\nFHFH\nFFFH\nHFFG\n\n(Right)\nS[F]FF\nFHFH\nFFFH\nHFFG\n\n(Right)\nSF[F]F\nFHFH\nFFFH\nHFFG\n\n(Right)\nSFFF\nFH[F]H\nFFFH\nHFFG\n\n(Right)\nSFFF\nFHF[H]\nFFFH\nHFFG\n\n(Right)\nSFFF\nFHF[4H]\nFFFH\nHFFG\n\n(Right)\nSFFF\nFHF[H]\nFFFH\nHFFG\n\n(Right)\nSFFF\nFHF[H]\nFFFH\nHFFG\n\n(Right)\nSFFF\nFHF[H]\nFFFH\nHFFG\n\n(Right)\nSFFF\nFHF[H]\nFFFH\nHFFG\n</code></pre>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#custom-frozen-lake-non-deterministic-policy-environment","title":"Custom Frozen Lake Non-deterministic Policy Environment","text":"<ul> <li>Because original code from OpenAI only allows us to run <code>env.step(action)</code>, this is challenging if we want to do some visualization of our state-value and action-value (q-value) functions for learning</li> <li>Hence, we'll be copying the whole code from OpenAI Frozen Lake implementation and adding just one line to make sure we can get P via <code>self.P = P</code></li> <li>This code is not important, you can just copy it</li> </ul> <pre><code>import sys\nfrom contextlib import closing\n\nimport numpy as np\nfrom six import StringIO, b\n\nfrom gym import utils\nfrom gym.envs.toy_text import discrete\n\nLEFT = 0\nDOWN = 1\nRIGHT = 2\nUP = 3\n\nMAPS = {\n    \"4x4\": [\n        \"SFFF\",\n        \"FHFH\",\n        \"FFFH\",\n        \"HFFG\"\n    ],\n    \"8x8\": [\n        \"SFFFFFFF\",\n        \"FFFFFFFF\",\n        \"FFFHFFFF\",\n        \"FFFFFHFF\",\n        \"FFFHFFFF\",\n        \"FHHFFFHF\",\n        \"FHFFHFHF\",\n        \"FFFHFFFG\"\n    ],\n}\n\n# Generates a random valid map (one that has a path from start to goal)\n# @params size, size of each side of the grid\n# @prams p, probability that a tile is frozen\ndef generate_random_map(size=8, p=0.8):\n    valid = False\n\n    #BFS to check that it's a valid path\n    def is_valid(arr, r=0, c=0):\n        if arr[r][c] == 'G':\n            return True\n\n        tmp = arr[r][c]\n        arr[r][c] = \"#\"\n\n        if r+1 &lt; size and arr[r+1][c] not in '#H':\n            if is_valid(arr, r+1, c) == True:\n                arr[r][c] = tmp\n                return True\n\n        if c+1 &lt; size and arr[r][c+1] not in '#H':\n            if is_valid(arr, r, c+1) == True:\n                arr[r][c] = tmp\n                return True\n\n        if r-1 &gt;= 0 and arr[r-1][c] not in '#H':\n            if is_valid(arr, r-1, c) == True:\n                arr[r][c] = tmp\n                return True\n\n        if c-1 &gt;= 0 and arr[r][c-1] not in '#H':\n            if is_valid(arr,r, c-1) == True:\n                arr[r][c] = tmp\n                return True\n        arr[r][c] = tmp\n        return False\n\n    while not valid:\n        p = min(1, p)\n        res = np.random.choice(['F','H'], (size, size), p=[p, 1-p])\n        res[0][0] = 'S'\n        res[-1][-1] = 'G'\n        valid = is_valid(res)\n    return [\"\".join(x) for x in res]\n\n\nclass FrozenLakeEnv(discrete.DiscreteEnv):\n    \"\"\"\n    Winter is here. You and your friends were tossing around a frisbee at the park\n    when you made a wild throw that left the frisbee out in the middle of the lake.\n    The water is mostly frozen, but there are a few holes where the ice has melted.\n    If you step into one of those holes, you'll fall into the freezing water.\n    At this time, there's an international frisbee shortage, so it's absolutely imperative that\n    you navigate across the lake and retrieve the disc.\n    However, the ice is slippery, so you won't always move in the direction you intend.\n    The surface is described using a grid like the following\n        SFFF\n        FHFH\n        FFFH\n        HFFG\n    S : starting point, safe\n    F : frozen surface, safe\n    H : hole, fall to your doom\n    G : goal, where the frisbee is located\n    The episode ends when you reach the goal or fall in a hole.\n    You receive a reward of 1 if you reach the goal, and zero otherwise.\n    \"\"\"\n\n    metadata = {'render.modes': ['human', 'ansi']}\n\n    def __init__(self, desc=None, map_name=\"4x4\",is_slippery=True):\n        if desc is None and map_name is None:\n            desc = generate_random_map()\n        elif desc is None:\n            desc = MAPS[map_name]\n        self.desc = desc = np.asarray(desc,dtype='c')\n        self.nrow, self.ncol = nrow, ncol = desc.shape\n        self.reward_range = (0, 1)\n\n        nA = 4\n        nS = nrow * ncol\n\n        isd = np.array(desc == b'S').astype('float64').ravel()\n        isd /= isd.sum()\n\n        P = {s : {a : [] for a in range(nA)} for s in range(nS)}\n\n        def to_s(row, col):\n            return row*ncol + col\n\n        def inc(row, col, a):\n            if a==0: # left\n                col = max(col-1,0)\n            elif a==1: # down\n                row = min(row+1,nrow-1)\n            elif a==2: # right\n                col = min(col+1,ncol-1)\n            elif a==3: # up\n                row = max(row-1,0)\n            return (row, col)\n\n        for row in range(nrow):\n            for col in range(ncol):\n                s = to_s(row, col)\n                for a in range(4):\n                    li = P[s][a]\n                    letter = desc[row, col]\n                    if letter in b'GH':\n                        li.append((1.0, s, 0, True))\n                    else:\n                        if is_slippery:\n                            for b in [(a-1)%4, a, (a+1)%4]:\n                                newrow, newcol = inc(row, col, b)\n                                newstate = to_s(newrow, newcol)\n                                newletter = desc[newrow, newcol]\n                                done = bytes(newletter) in b'GH'\n                                rew = float(newletter == b'G')\n                                li.append((1.0/3.0, newstate, rew, done))\n                        else:\n                            newrow, newcol = inc(row, col, a)\n                            newstate = to_s(newrow, newcol)\n                            newletter = desc[newrow, newcol]\n                            done = bytes(newletter) in b'GH'\n                            rew = float(newletter == b'G')\n                            li.append((1.0, newstate, rew, done))\n\n        # New change because environment only allows step without\n        # specific state for learning environment!\n        self.P = P\n\n        super(FrozenLakeEnv, self).__init__(nS, nA, P, isd)\n\n    def render(self, mode='human'):\n        outfile = StringIO() if mode == 'ansi' else sys.stdout\n\n        row, col = self.s // self.ncol, self.s % self.ncol\n        desc = self.desc.tolist()\n        desc = [[c.decode('utf-8') for c in line] for line in desc]\n        desc[row][col] = utils.colorize(desc[row][col], \"red\", highlight=True)\n        if self.lastaction is not None:\n            outfile.write(\"  ({})\\n\".format([\"Left\",\"Down\",\"Right\",\"Up\"][self.lastaction]))\n        else:\n            outfile.write(\"\\n\")\n        outfile.write(\"\\n\".join(''.join(line) for line in desc)+\"\\n\")\n\n        if mode != 'human':\n            with closing(outfile):\n                return outfile.getvalue()\n</code></pre>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#policy-evaluation","title":"Policy Evaluation","text":""},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#transition-probability-function","title":"Transition Probability Function","text":"<ul> <li>\\(\\mathcal{P}_{ss'}^a = \\mathcal{P}(s' \\vert s, a)  = \\mathbb{P} [S_{t+1} = s' \\vert S_t = s, A_t = a]\\)</li> </ul>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#deterministic-environment","title":"Deterministic Environment","text":"<ul> <li>There's no probability distribution, if you decide to go left you'll go left</li> <li>Hence in this example, given <code>current_state = 8</code> and <code>action = 0</code> which is left, we will end up with <code>probability = 1</code> in <code>new_state = 9</code></li> </ul> <pre><code># Deterministic\nenv = FrozenLakeEnv(is_slippery=False)\n\ncurrent_state = 10  # State from S_n=16 State space\naction = 0  # Left action from A_n=4 Action space\n[(probability, new_state, reward, done)] = env.P[current_state][action]\n\nprint('Probability {}, New State {}'.format(probability, new_state))\n</code></pre> <pre><code>Probability 1.0, New State 9\n</code></pre>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#stochastic-environment","title":"Stochastic Environment","text":"<ul> <li>Given \\(S_t = 10, A_t = 0\\) in a stochastic environment, the transition probability functions indicate you can end up in grid 6, 9, 14 each with \u2153 probability:<ul> <li>\\(\\mathbb{P} [S_{t+1} = 6 \\vert S_t = 10, A_t = 0] = \\frac{1}{3}\\)</li> <li>\\(\\mathbb{P} [S_{t+1} = 9 \\vert S_t = 10, A_t = 0] = \\frac{1}{3}\\)</li> <li>\\(\\mathbb{P} [S_{t+1} = 14 \\vert S_t = 10, A_t = 0] = \\frac{1}{3}\\)</li> </ul> </li> </ul> <pre><code># Stochastic\nenv = FrozenLakeEnv(is_slippery=True)\n\ncurrent_state = 10  # State from S_n=16 State space\naction = 0  # Left action from A_n=4 Action space\nenv.P[current_state][action]\n</code></pre> <pre><code>[(0.3333333333333333, 6, 0.0, False),\n (0.3333333333333333, 9, 0.0, False),\n (0.3333333333333333, 14, 0.0, False)]\n</code></pre>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#random-policy-function","title":"Random Policy Function","text":"<p>Random Policy function</p> <pre><code># Random policy generation\ndef generate_random_policy(S_n, A_n):\n    # return np.random.randint(A_n, size=(S_n, A_n))\n    return np.ones([S_n, A_n]) / A_n\n\n# Given the total number of states S_n = 16\n# For each state out of 16 states, we can take 4 actions\n# Since this is a stochastic environment, we'll initialize a policy to have equal probabilities 0.25 of doing each action each state\npolicy = generate_random_policy(S_n, A_n)\nprint(policy.shape)\n</code></pre> <pre><code>(16, 4)\n</code></pre> <p>Policy plot</p> <pre><code>import seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nplt.figure(figsize=(5, 16))\nsns.heatmap(policy,  cmap=\"YlGnBu\", annot=True, cbar=False);\n</code></pre> <p></p>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#policy-evaluation-function-comprising-state-value-function","title":"Policy Evaluation Function comprising State-value Function","text":"<ul> <li>How: \\(\\mathcal{V}_{\\pi}(s) = \\sum_{a \\in \\mathcal{A}} \\pi(a | s) \\sum_{s' \\in \\mathcal{S}}  \\mathcal{P}_{ss'}^a \\big[\\mathcal{R}_s^a + \\gamma  {V}_{\\pi}(s')\\big]\\)<ul> <li>Simple code equation:<ul> <li>Values of state given policy = sum ( action probability * transition probability * [reward + discount * value of new state] )</li> </ul> </li> </ul> </li> <li>Aim: getting state-values</li> </ul> <pre><code>import numpy as np\n\ndef policy_evaluation(env, policy, gamma=1., theta=1e-8):\n    r\"\"\"Policy evaluation function. Loop until state values stable, delta &lt; theta.\n\n    Returns V comprising values of states under given policy.\n\n    Args:\n        env (gym.env): OpenAI environment class instantiated and assigned to an object.\n        policy (np.array): policy array to evaluate\n        gamma (float): discount rate for rewards\n        theta (float): tiny positive number, anything below it indicates value function convergence\n    \"\"\"\n    # 1. Create state-value array (16,)\n    V = np.zeros(S_n)\n    while True:\n        delta = 0\n\n        # 2. Loop through states\n        for s in range(S_n):\n            Vs = 0\n\n            # 2.1 Loop through actions for the unique state\n            # Given each state, we've 4 actions associated with different probabilities\n            # 0.25 x 4 in this case, so we'll be looping 4 times (4 action probabilities) at each state\n            for a, action_prob in enumerate(policy[s]):\n                # 2.1.1 Loop through to get transition probabilities, next state, rewards and whether the game ended\n                for prob, next_state, reward, done in env.P[s][a]:\n                    # State-value function to get our values of states given policy\n                    Vs += action_prob * prob * (reward + gamma * V[next_state])\n\n            # This simple equation allows us to stop this loop when we've converged\n            # How do we know? The new value of the state is smaller than a tiny positive value we set\n            # State value change is tiny compared to what we have so we just stop!\n            delta = max(delta, np.abs(V[s]-Vs))\n\n            # 2.2 Update our state value for that state\n            V[s] = Vs\n\n        # 3. Stop policy evaluation if our state values changes are smaller than our tiny positive number\n        if delta &lt; theta:\n            break\n\n    return V\n\n# Generate random policy with equal probabilities of each action given any state\nrand_policy = generate_random_policy(S_n, A_n)\n\n# Evaluate the policy to get state values\nV = policy_evaluation(env, rand_policy)\n\n# Plot heatmap\nplt.figure(figsize=(8, 8))\nsns.heatmap(V.reshape(4, 4),  cmap=\"YlGnBu\", annot=True, cbar=False);\n</code></pre> <pre><code># This is our environment\n# Notice how the state values near the goal have higher values?\n# Those with \"H\" = hole, where you die if you step, have 0 values indicating those are bad areas to be in\nenv.render()\n</code></pre> <pre><code>[S]FFF\nFHFH\nFFFH\nHFFG\n</code></pre>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#policy-improvement","title":"Policy Improvement","text":""},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#action-value-q-value-function-from-state-value-function","title":"Action-value (Q-value) function from State-value function","text":"<ul> <li>How: \\(\\mathcal{Q}_{\\pi}(s, a) = \\sum_{s' \\in \\mathcal{S}} \\mathcal{P}_{ss'}^a \\big[ \\mathcal{R}_s^a + \\gamma  \\mathcal{V}_{\\pi}(s') \\big]\\)<ul> <li>Code equation<ul> <li>Values of action = sum ( transition probability * [reward + discount * value of next state] )</li> </ul> </li> </ul> </li> <li>Aim: getting q-values (action-values)</li> </ul> <pre><code>def q_value(env, V, s, gamma=1):\n    r\"\"\"Q-value (action-value) function from state-value function\n\n    Returns Q values, values of actions.\n\n    Args:\n        env (gym.env): OpenAI environment class instantiated and assigned to an object.\n        V (np.array): array of state-values obtained from policy evaluation function.\n        s (integer): integer representing current state in the gridworld\n        gamma (float): discount rate for rewards.\n    \"\"\"\n    # 1. Create q-value array for one state\n    # We have 4 actions, so let's create an array with the size of 4\n    q = np.zeros(A_n)\n\n    # 2. Loop through each action\n    for a in range(A_n):\n        # 2.1 For each action, we've our transition probabilities, next state, rewards and whether the game ended\n        for prob, next_state, reward, done in env.P[s][a]:\n            # 2.1.1 Get our action-values from state-values\n            q[a] += prob * (reward + gamma * V[next_state])\n\n    # Return action values\n    return q\n\n# For every state, we've 4 actions, hence we've 16 x 4 q values\nQ = np.zeros([S_n, A_n])\n\n# Loop through each state out of 16\n# For each state, we will get the 4 q-values associated with the 4 actions\nfor s in range(env.nS):\n    Q[s] = q_value(env, V, s)\n\nplt.figure(figsize=(5, 16))\nsns.heatmap(Q,  cmap=\"YlGnBu\", annot=True, cbar=False);\n</code></pre> <pre><code># Notice how 13/14, those in the last row of the gridworld just before reaching the goal of finishing the game, their action values are large?\nenv.render()\n</code></pre> <pre><code>[S]FFF\nFHFH\nFFFH\nHFFG\n</code></pre>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#policy-improvement-function","title":"Policy Improvement Function","text":"<ul> <li>How: maximizing q-values per state by choosing actions with highest q-values</li> <li>Aim: get improved policy</li> </ul> <pre><code>def policy_improvement(env, V, gamma=1.):\n    r\"\"\"Function to improve the policy by utilizing state values and action (q) values.\n\n    Args:\n        env (gym.env): OpenAI environment class instantiated and assigned to an objects\n        V (np.array): array of state-values obtained from policy evaluation function\n        gamma (float): discount of rewards\n    \"\"\"\n    # 1. Blank policy\n    policy = np.zeros([env.nS, env.nA]) / env.nA\n\n    # 2. For each state in 16 states\n    for s in range(env.nS):\n\n        # 2.1 Get q values: q.shape returns (4,)\n        q = q_value(env, V, s, gamma)\n\n        # 2.2 Find best action based on max q-value\n        # np.argwhere(q==np.max(q)) gives the position of largest q value\n        # given array([0.00852356, 0.01163091, 0.0108613 , 0.01550788]), this would return array([[3]]) of shape (1, 1)\n        # .flatten() reduces the shape to (1,) where we've array([3])\n        best_a = np.argwhere(q==np.max(q)).flatten()\n\n        # 2.3 One-hot encode best action and store into policy array's row for that state\n        # In our case where the best action is array([3]), this would return\n        # array([0., 0., 0., 1.]) where position 3 is the best action\n        # Now we can store the best action into our policy\n        policy[s] = np.sum([np.eye(env.nA)[i] for i in best_a], axis=0)/len(best_a)\n\n    return policy\n\n\nnew_policy = policy_improvement(env, V)\n\nplt.figure(figsize=(5, 16))\nsns.heatmap(new_policy,  cmap=\"YlGnBu\", annot=True, cbar=False);\n</code></pre> <pre><code># Compared to this equiprobable policy, the one above is making some improvements by maximizing q-values per state\nplt.figure(figsize=(5, 16))\nsns.heatmap(rand_policy,  cmap=\"YlGnBu\", annot=True, cbar=False);\n</code></pre>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#policy-iteration-function","title":"Policy Iteration Function","text":"<ul> <li>How: loop through policy evaluation (get state-values) and policy improvement functions (use state-values to calculate q-values to improve policy) until optimal policy obtained</li> <li>Aim: improve policy until convergence<ul> <li>Convergence: difference of state values between old and new policies is very small (less than theta, a very small positive number)</li> </ul> </li> </ul> <pre><code>import copy\ndef policy_iteration(env, gamma=1, theta=1e-8):\n    # 1. Create equiprobable policy where every state has 4 actions with equal probabilities as a starting policy\n    policy = np.ones([env.nS, env.nA]) / env.nA\n\n    # 2. Loop through policy_evaluation and policy_improvement functions\n    while True:\n        # 2.1 Get state-values\n        V = policy_evaluation(env, policy, gamma, theta)\n\n        # 2.2 Get new policy by getting q-values and maximizing q-values per state to get best action per state\n        new_policy = policy_improvement(env, V)\n\n        # 2.3 Stop if the value function estimates for successive policies has converged\n        if np.max(abs(policy_evaluation(env, policy) - policy_evaluation(env, new_policy))) &lt; theta * 1e2:\n            break;\n\n        # 2.4 Replace policy with new policy\n        policy = copy.copy(new_policy)\n    return policy, V\n\n# obtain the optimal policy and optimal state-value function\npolicy_pi, V_pi = policy_iteration(env)\n\n# Optimal policy (pi) \n# LEFT = 0, DOWN = 1, RIGHT = 2, UP = 3\nplt.figure(figsize=(5, 16))\nsns.heatmap(policy_pi,  cmap=\"YlGnBu\", annot=True, cbar=False, square=True);\n</code></pre> <pre><code># State values\nplt.figure(figsize=(8, 8))\nsns.heatmap(V_pi.reshape(4, 4),  cmap=\"YlGnBu\", annot=True, cbar=False, square=True);\n</code></pre> <pre><code># State values without policy improvement, just evaluation\nplt.figure(figsize=(8, 8))\nsns.heatmap(V.reshape(4, 4),  cmap=\"YlGnBu\", annot=True, cbar=False);\n</code></pre>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/#value-iteration","title":"Value iteration","text":"<ul> <li>Alternative to policy iteration</li> <li>How: loop through to find optimal value function then get one-off policy</li> <li>Aim: improve value function until convergence<ul> <li>Convergence: until difference in new and old state values are small (smaller than theta, small positive number)</li> </ul> </li> </ul> <pre><code>def value_iteration(env, gamma=1, theta=1e-8):\n    # 1. Create state values of shape (16,)\n    V = np.zeros(env.nS)\n\n    # 2. Loop through q-value function until convergence\n    while True:\n        delta = 0\n\n        # 2.1 Loop through each state\n        for s in range(env.nS):\n            # 2.2 Archive old state value\n            v = V[s]\n\n            # 2.3 New state value = max of q-value\n            V[s] = max(q_value(env, V, s, gamma))\n\n            delta = max(delta, abs(V[s] - v))\n\n        # 2.2 If state value changes small, converged\n        if delta &lt; theta:\n            break\n\n    # 3. Extract one-off policy with optimal state values\n    policy = policy_improvement(env, V, gamma)\n\n    return policy, V\n\npolicy_vi, V_vi = value_iteration(env)\n\n# Optimal policy\n# LEFT = 0, DOWN = 1, RIGHT = 2, UP = 3\nplt.figure(figsize=(5, 16))\nsns.heatmap(policy_vi,  cmap=\"YlGnBu\", annot=True, cbar=False, square=True);\n</code></pre> <pre><code># State values\nplt.figure(figsize=(8, 8))\nsns.heatmap(V_vi.reshape(4, 4),  cmap=\"YlGnBu\", annot=True, cbar=False, square=True);\n</code></pre>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/supervised_to_rl/","title":"Supervised Learning to Reinforcement Learning","text":""},{"location":"deep_learning/deep_reinforcement_learning_pytorch/supervised_to_rl/#supervised-learning","title":"Supervised Learning","text":"<ul> <li>The tasks we've covered so far fall under the category of supervised learning<ul> <li>Before, we have gone through 2 major tasks: classification and regression with labels</li> <li>Classification: we've a number of MNIST images, we take them as input and we use a neural network for a classification task where we use the ground truth (whether the digits are 0-9) to construct our cross entropy loss<ul> <li>Classification loss function (cross entropy loss): \\(- \\sum^K_1 L_i log(S_i)\\)<ul> <li>\\(K\\): number of classes</li> <li>\\(L_i\\): ground truth (label/target) of i-th class</li> <li>\\(S_i\\): output of softmax for i-th class</li> </ul> </li> </ul> </li> <li>Regression: alternatively we go through a regression task of say, predicting a time-series, but we still have the ground truth we use to construct our loss function<ul> <li>Regression loss function (mean squared error): \\(\\frac{1}{n} \\sum_{i=1}^n(\\hat y_i - y_i)^2\\)<ul> <li>\\(\\hat{y}\\): prediction</li> <li>\\(y\\): ground truth (label/target)</li> </ul> </li> </ul> </li> </ul> </li> <li>The key emphasis here is that we have mainly gone through supervised learning tasks that requires labels. Without them, we would not be able to properly construct our loss functions for us to do 2 critical steps (1) backpropagate to get our gradients and (2) gradient descent to update our weights with our gradients</li> </ul> <p>Loss functions</p> <p>We have covered 2 basic loss functions such as cross entropy loss (classification task) and mean squared error (regression task).</p> <p>However there are many more loss functions for both classification and regression task that will be covered in a separate section. </p> <p>For example, there are alternatives to mean squared error (MSE) like mean absolute error (MAE or L1 Loss), Smooth L1 Loss (less sensitive to outliers), quantile regression loss function (allowing confidence intervals) and many more.</p>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/supervised_to_rl/#reinforcement-learning-is-not-supervised-learning","title":"Reinforcement learning is not supervised learning","text":"<ul> <li>One difference is that there is no ground truth (label/target)<ul> <li>There is typically no label as to what is the definitively right prediction, we have to explore to find out what's \"right\" (essentially, the best possible prediction)</li> </ul> </li> <li>Instead of minimizing a loss function comprising a target and a prediction (as with supervised learning), in reinforcement learning we are typically concerned with maximizing our reward function by trying different actions and exploring what those actions yield in an environment<ul> <li>Let's use a simple game example of driving and not colliding with puppies crossing the road<ul> <li>Agent<ul> <li>Driver</li> </ul> </li> <li>Environment<ul> <li>3 lane road, puppies crossing and the agent</li> </ul> </li> <li>States<ul> <li>Left, center or right lane</li> </ul> </li> <li>Actions<ul> <li>To move from one state to another</li> <li>Turn left, center or right</li> </ul> </li> <li>Reward<ul> <li>Feedback on whether action is good/bad, essentially the goal of the problem</li> <li>Colliding with the puppy: -10 points</li> <li>Too close to the puppy (scares the puppy): -2 points</li> <li>Safe distance from the puppy: 10 points</li> </ul> </li> <li>Value function<ul> <li>Defines what is good in the long-run as compared to rewards which is immediate after an action takes the agent to another state</li> <li>It's somewhat the discounted sum of the rewards the agent is expected to get</li> </ul> </li> <li>Policy<ul> <li>This defines how the agent acts in its states</li> </ul> </li> <li>In this case, the agent might first collide with the puppy and learn it's bad (-10 points), then try not collide as the second action and still learn it's bad to be too close (-2 points) and finally as the third action learn to steer clear of puppies (+10 points) as it yields the largest reward<ul> <li>Gradually it'll learn to drive at a safe distance from puppies to collect points (+10 points for safe distance)</li> <li>To do this, the agent needs to go try different actions and learn from its mistakes (trial and error), attempting to maximize its long-term</li> </ul> </li> </ul> </li> </ul> </li> </ul>"},{"location":"deep_learning/deep_reinforcement_learning_pytorch/supervised_to_rl/#2-distinguishing-properties-of-reinforcement-learning","title":"2 Distinguishing Properties of reinforcement learning","text":"<ul> <li>Essentially, 2 distinguishing properties of reinforcement learning are: <sup>1</sup><ul> <li>(1) \"Trial-and-error search\"</li> <li>(2) \"Delayed reward\"</li> </ul> </li> </ul> <p>In the next section, we'll be covering the terms we'll dive into these key terms through the lens of Markov Decision Processes (MDPs) and Bellman Equations.</p> <ol> <li> <p>Richard S. Sutton and Andrew G. Barto. Reinforcement Learning: An Introduction; 2<sup>nd</sup> Edition. 2017.\u00a0\u21a9</p> </li> </ol>"},{"location":"deep_learning/fromscratch/fromscratch_cnn/","title":"CNN from Scratch","text":"<p>This is an implementation of a simple CNN (one convolutional function, one non-linear function, one max pooling function, one affine function and one softargmax function) for a 10-class MNIST classification task.</p>"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#data-pull","title":"Data Pull","text":""},{"location":"deep_learning/fromscratch/fromscratch_cnn/#imports","title":"Imports","text":"<pre><code>from sklearn.datasets import fetch_openml\nfrom sklearn.utils import check_random_state\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nimport torch\n%matplotlib inline\n</code></pre>"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#load-data","title":"Load Data","text":"<pre><code># Load data from https://www.openml.org/d/554\nX, y = fetch_openml('mnist_784', version=1, return_X_y=True)\n</code></pre> <pre><code>### Plot Single Sample\n</code></pre> <pre><code># Print dataset\nprint(f'X shape: {X.shape}')\nprint(f'y shape: {y.shape}')\n\n# Print size of resized single sample\nsingle_sample = X[0, :].reshape(28, 28)\nprint(f'Single sample shape: {single_sample.shape}')\n\n# Plot single sample (M x N matrix)\nplt.gray()\nplt.matshow(single_sample)\nplt.show()\n</code></pre> <pre><code>X shape: (70000, 784)\ny shape: (70000,)\nSingle sample shape: (28, 28)\n\n\n\n&lt;Figure size 432x288 with 0 Axes&gt;\n</code></pre>"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#train-test-data-split","title":"Train-Test Data Split","text":"<pre><code># Train-test split\ntrain_samples = 60000\nrandom_state = check_random_state(0)\npermutation = random_state.permutation(X.shape[0])\nX = X[permutation]\ny = y[permutation]\nX = X.reshape((X.shape[0], -1))\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, train_size=train_samples, test_size=10000)\n\nscaler = StandardScaler()\nX_train = scaler.fit_transform(X_train)\nX_test = scaler.transform(X_test)\n\n# Print shapes\nprint(f'Training shape {X_train.shape}')\nprint(f'Testing shape {X_test.shape}')\n</code></pre> <pre><code>Training shape (60000, 784)\nTesting shape (10000, 784)\n</code></pre>"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#forwardpropagation","title":"Forwardpropagation","text":""},{"location":"deep_learning/fromscratch/fromscratch_cnn/#convolutional-layer","title":"Convolutional Layer","text":"<pre><code>class ConvolutionalLayer:\n    def __init__(self, num_kernels, kernel_shape):\n        # Number of kernels: 1D\n        self.num_kernels = num_kernels\n        # Shape of kernels: 2D\n        self.kernel_shape = kernel_shape\n        self.k = self.kernel_shape[0]\n        # Kernel weights: 3D\n        self.kernels_theta = torch.randn(self.num_kernels, self.kernel_shape[0], self.kernel_shape[1])\n\n    def slider(self, inp):\n        '''\n        Sliding generator that yields square areas of shape\n        (kernel_shape, kernel_shape) sliding across our input. \n        This assumes valid padding (no padding) and step size 1.\n        '''\n        h, w = inp.shape\n        for h_idx in range(h - (self.k - 1)):\n            for w_idx in range(w - (self.k - 1)):\n                single_slide_area = inp[h_idx:(h_idx + self.k), w_idx:(w_idx + self.k)]\n                yield single_slide_area, h_idx,w_idx\n\n    def forward(self, inp):\n        '''\n        Slides kernel across image doing an element-wise MM then summing.\n        Results in forward pass of convolutional layer of shape\n        (output shape, output shape, number of kernels).\n        '''\n        # Input: 2D of (height, width)\n        assert single_sample.dim() == 2, f'Input not 2D, given {single_sample.dim()}D'\n\n        # Output via Valid Padding (No Padding): 3D of (height, width, number of kernels)\n        _, w  = inp.shape\n        # P = 0\n        p = 0\n        # O = ((W - K + 2P) / S) + 1 = (28 - 3 + 0) + 1 = 25\n        o = (w - self.k) + 1\n        # Print shapes\n        print('Padding shape: \\t', p)\n        print('Output shape: \\t', o)\n        # Initialize blank tensor\n        output = torch.zeros((o, o, self.num_kernels))\n\n        # Iterate through region\n        for single_slide_area, h_idx, w_idx in self.slider(inp):\n            if h_idx == 0 and w_idx == 0:\n                print('Region shape: \\t', list(single_slide_area.shape))\n                print('Kernel shape: \\t', list(self.kernels_theta.shape))\n                print('Single Slide: \\t', list(output[h_idx, w_idx].shape))\n\n            # Sum values with each element-wise matrix multiplication across each kernel\n            # Instead of doing another loop of each kernel, you simply just do a element-wise MM\n            # of the single slide area with all the kernels yield, then summing the patch\n            output[h_idx, w_idx] = torch.sum(single_slide_area * self.kernels_theta, axis=(1, 2))\n\n        # Pass through non-linearity (sigmoid): 1 / 1 + exp(-output)\n        output = 1. / (1. + torch.exp(-output))\n\n        return output\n\n# Convert numpy array to torch tensor\nsingle_sample = X[0, :].reshape(28, 28)\nsingle_sample = torch.tensor(single_sample)\n\nprint('='*50)\nprint(f'Input shape: \\t {list(single_sample.shape)}')\nprint('='*50)\n\n# Forward: conv\nconv = ConvolutionalLayer(num_kernels=8, kernel_shape=[5, 5])\noutput = conv.forward(single_sample)\n\nprint('='*50)\nprint(f'Conv (f) shape: \\t {list(output.shape)}')\nprint('='*50)\n</code></pre> <pre><code>==================================================\nInput shape:     [28, 28]\n==================================================\nPadding shape:   0\nOutput shape:    24\nRegion shape:    [5, 5]\nKernel shape:    [8, 5, 5]\nSingle Slide:    [8]\n==================================================\nConv (f) shape:      [24, 24, 8]\n==================================================\n</code></pre>"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#max-pooling-layer","title":"Max Pooling Layer","text":"<pre><code>class MaxPoolLayer:\n    # O = ((W - K) / S) + 1\n    def __init__(self, pooling_kernel_shape):\n        # Assume simplicity of K = S then O = W / S\n        self.k = pooling_kernel_shape\n\n    def slider(self, inp):\n        '''\n        Sliding generator that yields areas for max pooling.\n        '''\n        h, w, _ = inp.shape\n        output_size = int(w / self.k)  # Assume S = K\n\n        for h_idx in range(output_size):\n            for w_idx in range(output_size):\n                single_slide_area = inp[h_idx * self.k:h_idx * self.k + self.k,\n                                        w_idx * self.k:w_idx * self.k + self.k]\n                yield single_slide_area, h_idx, w_idx\n\n    def forward(self, inp):\n        '''\n        Performs a forward pass of the maxpool layer using the given input.\n        Returns a 3d numpy array with dimensions (h / 2, w / 2, num_filters).\n        - input is a 3d numpy array with dimensions (h, w, num_filters)\n        '''\n        self.last_input = inp\n\n        h, w, num_kernels = inp.shape\n        output_size = int(w / self.k)  # Assume S = K\n        output = torch.zeros(output_size, output_size, num_kernels)\n\n        for single_slide_area, h_idx, w_idx in self.slider(inp):\n            single_slide_area = torch.flatten(single_slide_area, start_dim=0, end_dim=1)\n            output[h_idx, w_idx] = torch.max(single_slide_area, dim=0).values\n\n        return output\n\nprint('='*50)\nprint(f'Input shape: \\t {list(output.shape)}')\nprint('='*50)\n\n# Forward: pool\npool = MaxPoolLayer(pooling_kernel_shape=2)\noutput = pool.forward(output)\n\nprint('='*50)\nprint(f'Pool (f) shape: \\t {list(output.shape)}')\nprint('='*50)\n</code></pre> <pre><code>==================================================\nInput shape:     [24, 24, 8]\n==================================================\n==================================================\nPool (f) shape:      [12, 12, 8]\n==================================================\n</code></pre>"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#affine-and-softargmax-layer","title":"Affine and Soft(arg)max Layer","text":"<pre><code>class AffineAndSoftmaxLayer:\n    def __init__(self, affine_weight_shape):\n        self.affine_weight_shape = affine_weight_shape\n        # Weight shape: flattened input x output shape\n        self.w = torch.zeros(self.affine_weight_shape[0] * self.affine_weight_shape[1] * self.affine_weight_shape[2], self.affine_weight_shape[3])\n        self.b = torch.zeros(self.affine_weight_shape[3])\n\n        # Initialize weight/bias via Lecun initialization of 1 / N standard deviation\n        # Refer to DLW guide on weight initialization mathematical derivation:\n        # https://www.deeplearningwizard.com/deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/\n        print(f'Lecun initialization SD: {1/self.affine_weight_shape[3]}')\n        self.w = torch.nn.init.normal_(self.w, mean=0, std=1/self.affine_weight_shape[3])\n        self.b = torch.nn.init.normal_(self.b, mean=0, std=1/self.affine_weight_shape[3])\n\n    def forward(self, inp):\n        '''\n        Performs Linear (Affine) Function &amp; Soft(arg)max Function\n        that returns our vector (1D) of probabilities.\n        '''\n        inp = inp.reshape(1, -1)\n        print(f'input shape: \\t {inp.shape}')\n        print(f'weight shape: \\t {self.w.shape}')\n        print(f'bias shape: \\t {self.b.shape}')\n        logits = torch.mm(inp, self.w) + self.b\n        probas = torch.exp(logits) / torch.sum(torch.exp(logits))\n        return probas\n\n\nprint('='*50)\nprint(f'Input shape: \\t {list(output.shape)}')\nprint('='*50)\n\n# Forward: Affine and Softmax\naffinesoftmax = AffineAndSoftmaxLayer(affine_weight_shape=(output.shape[0], output.shape[1], output.shape[2], len(np.unique(y_train))))\noutput = affinesoftmax.forward(output)\n\nprint('='*50)\nprint(f'Affine &amp; Soft(arg)max (f) shape: \\t {list(output.shape)}')\nprint('='*50)\n\nprint(f'Probas: {pd.DataFrame(output.numpy()).to_string(index=False, header=False)}')\n</code></pre> <pre><code>==================================================\nInput shape:     [12, 12, 8]\n==================================================\nLecun initialization SD: 0.1\ninput shape:     torch.Size([1, 1152])\nweight shape:    torch.Size([1152, 10])\nbias shape:      torch.Size([10])\n==================================================\nAffine &amp; Soft(arg)max (f) shape:     [1, 10]\n==================================================\nProbas:  0.72574  0.001391  0.000083  0.202983  0.000503  0.037023  0.000428  0.000144  0.031293  0.000412\n</code></pre> <p>Dot Product, Matrix Multiplication, and Hadamard Product</p> <p>Hadamard product: element-wise multiplicaton of 2 matrices. Matrix Multiplication: take the first row of the first matrix and perform dot product with each of the N columns in the second matrix to form N columns in the first row of the new matrix. Repeat for remaining rows for the first matrix.    </p> <pre><code># Backward &amp; GD: linear + softmax\n\n# Backward &amp; GD: pool\n\n# Backward &amp; GD: conv\n</code></pre>"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#note-on-contiguous-vs-non-contiguous","title":"Note on Contiguous vs Non-Contiguous","text":"<ul> <li>Often in the space of passing tensors, and doing all our dot products, hadamard products, matrix multiplications, transpose, reshape operations and more, you would inevitably one day encounter into an error that says your tensor is <code>not contiguous</code>. </li> <li>This is a memory allocation problem. </li> <li>Certain tensor operations like <code>transpose, view, expand, narrow etc</code> do not change the original tensor, instead they modify the properties of the tensor. <ul> <li>For example, <code>transpose</code>, demonstrated here, would change the shape (index), but both the old and modified property tensor share the same memory block with different indexes/addresses.</li> <li>This is why it's non-contiguous and we need to make it contiguous for some operations and typically for efficiency purpose.</li> </ul> </li> <li>This is not a blanket statement, but you typically want your tensors to be contiguous as it prevents additional overhead incurred from translating addresses.<ul> <li>Whenever there's a warning that prompts you the tensor is not contiguous, just call <code>.contiguous()</code> and you typically should be good to go.</li> </ul> </li> </ul>"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#contiguous-10-by-5-tensor","title":"Contiguous 10 by 5 tensor","text":"<pre><code>contiguous_tensor = torch.arange(50).view(10, 5)\nprint(contiguous_tensor.shape)\n# Pretty print quick hack via Pandas DataFrame\nprint(pd.DataFrame(contiguous_tensor.numpy()).to_string(index=False, header=False))\n</code></pre> <pre><code>torch.Size([10, 5])\n  0   1   2   3   4\n  5   6   7   8   9\n 10  11  12  13  14\n 15  16  17  18  19\n 20  21  22  23  24\n 25  26  27  28  29\n 30  31  32  33  34\n 35  36  37  38  39\n 40  41  42  43  44\n 45  46  47  48  49\n</code></pre>"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#stride-5-by-1","title":"Stride 5 by 1","text":"<ul> <li>Stride here shows how we need:<ul> <li>5 steps to move one row to the next &amp;</li> <li>1 step to move from one column to the next (contiguous)<ul> <li>If this becomes anything other than 1 step, it becomes not contiguous</li> </ul> </li> </ul> </li> </ul> <pre><code>print(contiguous_tensor.stride())\nprint(contiguous_tensor.is_contiguous())\n</code></pre> <pre><code>(5, 1)\nTrue\n</code></pre>"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#non-contiguous-tensor-via-transpose-operation","title":"Non-contiguous Tensor via Transpose Operation","text":"<ul> <li>In order to access the next \"column\" value of 5, we have to take 5 steps despite the transpose as if we would do in our original tensor</li> <li>Because the original tensor and this transposed tensor share the same memory block!</li> </ul> <pre><code>non_contiguous_tensor = contiguous_tensor.t()\nprint(non_contiguous_tensor.shape)\nprint(pd.DataFrame(non_contiguous_tensor.numpy()).to_string(index=False, header=False))\n</code></pre> <pre><code>torch.Size([5, 10])\n 0  5  10  15  20  25  30  35  40  45\n 1  6  11  16  21  26  31  36  41  46\n 2  7  12  17  22  27  32  37  42  47\n 3  8  13  18  23  28  33  38  43  48\n 4  9  14  19  24  29  34  39  44  49\n</code></pre>"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#stride-1-by-5","title":"Stride 1 by 5","text":"<pre><code>print(non_contiguous_tensor.stride())\nprint(non_contiguous_tensor.is_contiguous())\n</code></pre> <pre><code>(1, 5)\nFalse\n</code></pre>"},{"location":"deep_learning/fromscratch/fromscratch_cnn/#convert-to-contiguous","title":"Convert to Contiguous","text":"<pre><code># Convert to contiguous\nconvert_contiguous = non_contiguous_tensor.contiguous()\nprint(convert_contiguous.is_contiguous())\n</code></pre> <pre><code>True\n</code></pre>"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/","title":"Logistic Regression from Scratch","text":"<p>This is an implementation of a simple logistic regression for binary class labels. We will be attempting to classify 2 flowers based on their petal width and height: setosa and versicolor.</p>"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#imports","title":"Imports","text":"<pre><code>%matplotlib inline\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nimport torch\nimport torch.nn.functional as F\n\nfrom sklearn import datasets\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import train_test_split\nfrom collections import Counter\n</code></pre>"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#preparing-a-custom-2-class-iris-dataset","title":"Preparing a custom 2-class IRIS dataset","text":""},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#load-data","title":"Load Data","text":"<pre><code># Instantiate dataset class and assign to object\niris = datasets.load_iris()\n\n# Load features and target\n# Take only 2 classes, and 2 features (sepal length/width)\nX = iris.data[:-50, :2]\n# For teaching the math rather than preprocessing techniques,\n# we'll be using this simple scaling method. However, you must\n# be cautious to scale your training/testing sets subsequently.\nX = preprocessing.scale(X)\ny = iris.target[:-50] \n</code></pre>"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#print-data-details","title":"Print Data Details","text":"<pre><code># 50 of each iris flower\nprint(Counter(y))\n\n# Type of flower\nprint(list(iris.target_names[:-1]))\n\n# Shape of features\nprint(X.shape)\n</code></pre> <pre><code>Counter({0: 50, 1: 50})\n['setosa', 'versicolor']\n(100, 2)\n</code></pre>"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#scatterplot-2-classes","title":"Scatterplot 2 Classes","text":"<pre><code>plt.scatter(X[:, 0], X[:, 1], c=y);\n</code></pre>"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#traintest-split","title":"Train/Test Split","text":"<pre><code>X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\nprint(f'X train size: {X_train.shape}')\nprint(f'X test size: {X_test.shape}')\nprint(f'y train size: {y_train.shape}')\nprint(f'y test size: {y_test.shape}')\n\n# Distribution of both classes are roughly equal using train_test_split function\nprint(Counter(y_train))\n</code></pre> <pre><code>X train size: (80, 2)\nX test size: (20, 2)\ny train size: (80,)\ny test size: (20,)\nCounter({0: 41, 1: 39})\n</code></pre>"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#math","title":"Math","text":""},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#1-forwardpropagation","title":"1. Forwardpropagation","text":"<ul> <li>Get our logits and probabilities</li> <li>Affine function/transformation: \\(z = \\theta x + b\\)</li> <li>Sigmoid/logistic function: \\(\\hat y = \\frac{1}{1 + e^{-z}}\\)</li> </ul>"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#2-backwardpropagation","title":"2. Backwardpropagation","text":"<ul> <li>Calculate gradients / partial derivatives w.r.t. weights and bias</li> <li>Loss: \\(L = ylog(\\hat y) + (1-y) log (1 - \\hat y)\\)</li> <li>Partial derivative of loss w.r.t weights: \\(\\frac{\\delta L}{\\delta w} =\\frac{\\delta L}{\\delta z} \\frac{\\delta z}{\\delta w} = (\\hat y - y)(x^T)\\)</li> <li>Partial derivative of loss w.r.t. bias: \\(\\frac{\\delta L}{\\delta b} = \\frac{\\delta L}{\\delta z} \\frac{\\delta z}{\\delta b} = (\\hat y - y)(1)\\)<ul> <li>\\(\\frac{\\delta L}{\\delta z} = \\hat y - y\\)</li> <li>\\(\\frac{\\delta z}{\\delta w} = x\\)</li> <li>\\(\\frac{\\delta z}{\\delta b} = 1\\)</li> </ul> </li> </ul>"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#2a-loss-function-clarification","title":"2a. Loss function clarification","text":"<ul> <li>Actually, why is our loss equation \\(L = ylog(\\hat y) + (1-y) log (1 - \\hat y)\\)?<ul> <li>We have given the intuition in the Logistic Regression tutorial on why it works.</li> <li>Here we will cover the derivation which essentially is merely maximizing the log likelihood of the parameters (maximizing the probability of our predicted output given our input and parameters</li> <li>Given:<ul> <li>\\(\\hat y = \\frac{1}{1 + e^{-z}}\\).</li> </ul> </li> <li>Then:<ul> <li>\\(P(y=1 \\mid x;\\theta) = \\hat y\\)</li> <li>\\(P(y=0 \\mid x;\\theta) = 1 - \\hat y\\)</li> </ul> </li> <li>Simplified further:<ul> <li>\\(p(y \\mid x; \\theta) = (\\hat y)^y(1 - \\hat y)^{1-y}\\)</li> </ul> </li> <li>Given m training samples, the likelihood of the parameters is simply the product of probabilities:<ul> <li>\\(L(\\theta) = \\displaystyle \\prod_{i=1}^{m} p(y^i \\mid x^i; \\theta)\\)</li> <li>\\(L(\\theta) = \\displaystyle \\prod_{i=1}^{m} (\\hat y^{i})^{y^i}(1 - \\hat y^{i})^{1-y^{i}}\\)</li> <li>Essentially, we want to maximize the probability of our ouput given our input and parameters</li> </ul> </li> <li>But it's easier to maximize the log likelihood, so we take the natural logarithm. <ul> <li>\\(L(\\theta) = \\displaystyle \\sum_{i=1}^{m} y^{i}log (\\hat y^{i}) + (1 - y^{i})log(1 - \\hat y^{i})\\)</li> </ul> </li> <li>Why is is easier to maximize the log likelihood?<ul> <li>The natural logarithm is a function that monotonically increases.</li> <li>This allows us to find the \"max\" of the log likelihood easier compared to a non-monotonically increasing function (like a wave up and down).</li> </ul> </li> </ul> </li> </ul>"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#3-gradient-descent-updating-weights","title":"3. Gradient descent: updating weights","text":"<ul> <li>\\(w = w - \\alpha (\\hat y - y)(x^T)\\)</li> <li>\\(b = b - \\alpha (\\hat y - y).1\\)</li> </ul>"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#training-from-scratch","title":"Training from Scratch","text":"<pre><code>learning_rate = 0.1\nnum_features = X.shape[1]\nweights = torch.zeros(num_features, 1, dtype=torch.float32)\nbias = torch.zeros(1, dtype=torch.float32)\n\nX_train = torch.from_numpy(X_train).type(torch.float32)\ny_train = torch.from_numpy(y_train).type(torch.float32)\n\nfor epoch in range(num_epochs):        \n    # 1. Forwardpropagation:\n    # 1a. Affine Transformation: z = \\theta x + b\n    z = torch.add(torch.mm(X_train, weights), bias)\n    # 2a. Sigmoid/Logistic Function: y_hat = 1 / (1 + e^{-z})\n    y_hat = 1. / (1. + torch.exp(-z))\n\n    # Backpropagation:\n    # 1. Calculate binary cross entropy \n    l = torch.mm(-y_train.view(1, -1), torch.log(y_hat)) - torch.mm((1 - y_train).view(1, -1), torch.log(1 - y_hat))\n\n    # 2. Calculate dl/dz\n    dl_dz = y_train - y_hat.view(-1)\n\n    # 2. Calculate partial derivative of cost w.r.t weights (gradients)\n    # dl_dw = dl_dz dz_dw = (y_hat - y)(x^T)\n    grad = torch.mm(X_train.transpose(0, 1), dl_dz.view(-1, 1))\n\n    # Gradient descent:\n    # update our weights and bias with our gradients\n    weights += learning_rate * grad\n    bias += learning_rate * torch.sum(dl_dz)\n\n    # Accuracy\n    total = y_hat.shape[0]\n    predicted = (y_hat &gt; 0.5).float().squeeze()\n    correct = (predicted == y_train).sum()\n    acc = 100 * correct / total \n\n    # Print accuracy and cost\n    print(f'Epoch: {epoch} | Accuracy: {acc.item() :.4f} | Cost: {l.item() :.4f}')\n\nprint(f'Weights \\n {weights.data}')\nprint(f'Bias \\n {bias.data}')\n</code></pre> <pre><code>X train size: (80, 2)\nX test size: (20, 2)\ny train size: (80,)\ny test size: (20,)\nCounter({1: 41, 0: 39})\nEpoch: 0 | Accuracy: 48.0000 | Cost: 55.4518\nEpoch: 1 | Accuracy: 100.0000 | Cost: 5.6060\nEpoch: 2 | Accuracy: 100.0000 | Cost: 5.0319\nEpoch: 3 | Accuracy: 100.0000 | Cost: 4.6001\nEpoch: 4 | Accuracy: 100.0000 | Cost: 4.2595\nEpoch: 5 | Accuracy: 100.0000 | Cost: 3.9819\nEpoch: 6 | Accuracy: 100.0000 | Cost: 3.7498\nEpoch: 7 | Accuracy: 100.0000 | Cost: 3.5521\nEpoch: 8 | Accuracy: 100.0000 | Cost: 3.3810\nEpoch: 9 | Accuracy: 100.0000 | Cost: 3.2310\nEpoch: 10 | Accuracy: 100.0000 | Cost: 3.0981\nEpoch: 11 | Accuracy: 100.0000 | Cost: 2.9794\nEpoch: 12 | Accuracy: 100.0000 | Cost: 2.8724\nEpoch: 13 | Accuracy: 100.0000 | Cost: 2.7754\nEpoch: 14 | Accuracy: 100.0000 | Cost: 2.6869\nEpoch: 15 | Accuracy: 100.0000 | Cost: 2.6057\nEpoch: 16 | Accuracy: 100.0000 | Cost: 2.5308\nEpoch: 17 | Accuracy: 100.0000 | Cost: 2.4616\nEpoch: 18 | Accuracy: 100.0000 | Cost: 2.3973\nEpoch: 19 | Accuracy: 100.0000 | Cost: 2.3374\nWeights \n tensor([[ 4.9453],\n        [-3.6849]])\nBias \n tensor([0.5570])\n</code></pre>"},{"location":"deep_learning/fromscratch/fromscratch_logistic_regression/#inference","title":"Inference","text":"<pre><code># Port to tensors\nX_test = torch.from_numpy(X_test).type(torch.float32)\ny_test = torch.from_numpy(y_test).type(torch.float32)\n\n# 1. Forwardpropagation:\n# 1a. Affine Transformation: z = ax + b\nz = torch.add(torch.mm(X_test, weights), bias)\n# 2a. Sigmoid/Logistic Function: y_hat = 1 / (1 + e^{-z})\ny_hat = 1. / (1. + torch.exp(-z))\n\ntotal = y_test.shape[0]\npredicted = (y_hat &gt; 0.5).float().squeeze()\ncorrect = (predicted == y_test).sum()\nacc = 100 * correct / total \n\n# Print accuracy\nprint(f'Validation Accuracy: {acc.item() :.4f}')\n</code></pre> <pre><code>Validation Accuracy: 100.0000\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_autoencoder/","title":"Autoencoders with PyTorch","text":"<p>Run Jupyter Notebook</p> <p>You can run the code for this section in this jupyter notebook link.</p>"},{"location":"deep_learning/practical_pytorch/pytorch_autoencoder/#about-autoencoders","title":"About Autoencoders","text":""},{"location":"deep_learning/practical_pytorch/pytorch_autoencoder/#feedforward-neural-network-fnn-to-autoencoders-aes","title":"Feedforward Neural Network (FNN) to Autoencoders (AEs)","text":"<ul> <li>Autoencoder is a form of unsupervised learning.<ul> <li>This is a big deviation from what we have been doing: classification and regression which are under supervised learning.</li> <li>There are no labels required, inputs are used as labels.<ul> <li>This is a bit mind-boggling for some, but there're many conrete use cases as you'll soon realize.</li> <li>Just a quick preview of use cases we will be covering:<ul> <li>Denoising overcomplete AEs: recreate images without the random noises originally injected.</li> <li>Undercomplete AEs for anomaly detection: use AEs for credit card fraud detection via anomaly detection.</li> <li>Variational AEs for creating synthetic faces: with a convolutional VAEs, we can make fake faces.</li> </ul> </li> </ul> </li> </ul> </li> <li>An autoencoder's purpose is to learn an approximation of the identity function (mapping \\(x\\) to \\(\\hat x\\)).<ul> <li>Essentially we are trying to learn a function that can take our input \\(x\\) and recreate it \\(\\hat x\\).<ul> <li>Technically we can do an exact recreation of our in-sample input if we use a very wide and deep neural network.</li> </ul> </li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_autoencoder/#undercomplete-and-overcomplete-autoencoders","title":"Undercomplete and Overcomplete Autoencoders","text":"<ul> <li>When we highlighted some use cases, did you notice how we mentioned undercomplete and autocomplete AEs?</li> <li>The only difference between the two is in the encoding output's size.<ul> <li>In the diagram above, this refers to the encoding output's size after our first affine function (yellow box) and non-linear function (pink box).</li> </ul> </li> <li>Undercomplete AEs: smaller<ul> <li>This is when our encoding output's dimension is smaller than our input's dimension.<ul> <li>Essentially we reduced the dimension of our data (dimensionality reduction) with an undercomplete AE</li> </ul> </li> </ul> </li> <li>Overcomplete AEs: larger<ul> <li>This is when our encoding output's dimension is larger than our input's dimension<ul> <li>Essentially we increased the dimension of our data with an overcomplete AE</li> </ul> </li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_autoencoder/#fully-connected-and-convolutional-autoencoders","title":"Fully-connected and Convolutional Autoencoders","text":"<ul> <li>Another important point is that, in our diagram we've used the example of our Feedforward Neural Networks (FNN) where we use fully-connected layers. <ul> <li>This is called Fully-connected AE.</li> </ul> </li> <li>However, we can easily swap those fully-connected layers with convolutional layers.<ul> <li>This is called Convolutional AE.</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_autoencoder/#autoencoders-series","title":"Autoencoders Series","text":"<ul> <li>We'll be covering a series of autoencoders in this order<ul> <li>Fully-connected Overcomplete Autoencoder (AEs): Denoising Images</li> <li>Fully-connected Undercomplete Autoencoder (AEs): Credit Card Fraud Detection</li> <li>Convolutional Overcomplete Variational Autoencoder (VAEs): Generate Fake Human Faces</li> <li>Convolutional Overcomplete Adversarial Autoencoder (AAEs): Generate Fake Human Faces</li> <li>Generative Adversarial Networks (GANs): Generate Better Fake Human Faces</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/","title":"Convolutional Neural Network with PyTorch","text":"<p>Run Jupyter Notebook</p> <p>You can run the code for this section in this jupyter notebook link.</p>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#about-convolutional-neural-network","title":"About Convolutional Neural Network","text":""},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#transition-from-feedforward-neural-network","title":"Transition From Feedforward Neural Network","text":""},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#hidden-layer-feedforward-neural-network","title":"Hidden Layer Feedforward Neural Network","text":"<p>Recap of FNN</p> <p>So let's do a recap of what we covered in the Feedforward Neural Network (FNN) section using a simple FNN with 1 hidden layer (a pair of affine function and non-linear function)</p> <ol> <li>[Yellow box] Pass input into an affine function \\(\\boldsymbol{y} = A\\boldsymbol{x} + \\boldsymbol{b}\\)</li> <li>[Pink box] Pass logits to non-linear function, for example sigmoid, tanh (hyperbolic tangent), ReLU, or LeakyReLU</li> <li>[Blue box] Pass output of non-linear function to another affine function</li> <li>[Red box] Pass output of final affine function to softmax function to get our probability distribution over K classes</li> <li>[Purple box] Finally we can get our loss by using our cross entropy function </li> </ol> <p></p>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#basic-convolutional-neural-network-cnn","title":"Basic Convolutional Neural Network (CNN)","text":"<ul> <li>A basic CNN just requires 2 additional layers!<ul> <li>Convolution and pooling layers before our feedforward neural network</li> </ul> </li> </ul> <p>Fully Connected (FC) Layer</p> <p>A layer with an affine function &amp; non-linear function is called a Fully Connected (FC) layer</p> <p></p>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#one-convolutional-layer-high-level-view","title":"One Convolutional Layer: High Level View","text":""},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#one-convolutional-layer-high-level-view-summary","title":"One Convolutional Layer: High Level View Summary","text":"<ul> <li>As the kernel is sliding/convolving across the image \\(\\rightarrow\\) 2 operations done per patch<ol> <li>Element-wise multiplication</li> <li>Summation</li> </ol> </li> <li>More kernels \\(=\\) more feature map channels<ul> <li>Can capture more information about the input</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#multiple-convolutional-layers-high-level-view","title":"Multiple Convolutional Layers: High Level View","text":""},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#pooling-layer-high-level-view","title":"Pooling Layer: High Level View","text":"<ul> <li>2 Common Types<ul> <li>Max Pooling</li> <li>Average Pooling</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#multiple-pooling-layers-high-level-view","title":"Multiple Pooling Layers: High Level View","text":""},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#padding","title":"Padding","text":""},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#padding-summary","title":"Padding Summary","text":"<ul> <li>Valid Padding (No Padding)<ul> <li>Output size &lt; Input Size</li> </ul> </li> <li>Same Padding (Zero Padding)<ul> <li>Output size = Input Size</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#dimension-calculations","title":"Dimension Calculations","text":"<ul> <li>\\(O = \\frac {W - K + 2P}{S} + 1\\)<ul> <li>\\(O\\): output height/length</li> <li>\\(W\\): input height/length</li> <li>\\(K\\): filter size (kernel size)</li> <li>\\(P\\): padding<ul> <li>\\(P = \\frac{K - 1}{2}\\)</li> </ul> </li> <li>\\(S\\): stride</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#example-1-output-dimension-calculation-for-valid-padding","title":"Example 1: Output Dimension Calculation for Valid Padding","text":"<ul> <li>\\(W = 4\\)</li> <li>\\(K = 3\\)</li> <li>\\(P = 0\\)</li> <li>\\(S = 1\\)</li> <li>\\(O = \\frac {4 - 3 + 2*0}{1} + 1 = \\frac {1}{1} + 1 = 1 + 1 = 2\\)</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#example-2-output-dimension-calculation-for-same-padding","title":"Example 2: Output Dimension Calculation for Same Padding","text":"<ul> <li>\\(W = 5\\)</li> <li>\\(K = 3\\)</li> <li>\\(P = \\frac{3 - 1}{2} = \\frac{2}{2} = 1\\)</li> <li>\\(S = 1\\)</li> <li>\\(O = \\frac {5 - 3 + 2*1}{1} + 1 = \\frac {4}{1} + 1 = 5\\)</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#building-a-convolutional-neural-network-with-pytorch","title":"Building a Convolutional Neural Network with PyTorch","text":""},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#model-a","title":"Model A:","text":"<ul> <li>2 Convolutional Layers<ul> <li>Same Padding (same output size)</li> </ul> </li> <li>2 Max Pooling Layers</li> <li>1 Fully Connected Layer</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#steps","title":"Steps","text":"<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#step-1-loading-mnist-train-dataset","title":"Step 1: Loading MNIST Train Dataset","text":"<p>Images from 1 to 9</p> <p>MNIST Dataset and Size of Training Dataset (Excluding Labels)</p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n</code></pre> <pre><code>train_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n</code></pre> <pre><code>print(train_dataset.train_data.size())\n</code></pre> <pre><code>torch.Size([60000, 28, 28])\n</code></pre> <p>Size of our training dataset labels</p> <pre><code>print(train_dataset.train_labels.size())\n</code></pre> <pre><code>torch.Size([60000])\n</code></pre> <p>Size of our testing dataset (excluding labels)</p> <pre><code>print(test_dataset.test_data.size())\n</code></pre> <pre><code>torch.Size([10000, 28, 28])\n</code></pre> <p>Size of our testing dataset labels</p> <pre><code>print(test_dataset.test_labels.size())\n</code></pre> <pre><code>torch.Size([10000])\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#step-2-make-dataset-iterable","title":"Step 2: Make Dataset Iterable","text":"<p>Load Dataset into Dataloader</p> <pre><code>batch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#step-3-create-model-class","title":"Step 3: Create Model Class","text":""},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#output-formula-for-convolution","title":"Output Formula for Convolution","text":"<ul> <li>\\(O = \\frac {W - K + 2P}{S} + 1\\)<ul> <li>\\(O\\): output height/length</li> <li>\\(W\\): input height/length</li> <li>\\(K\\): filter size (kernel size) = 5</li> <li>\\(P\\): same padding (non-zero)<ul> <li>\\(P = \\frac{K - 1}{2}  = \\frac{5 - 1}{2} = 2\\)</li> </ul> </li> <li>\\(S\\): stride = 1</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#output-formula-for-pooling","title":"Output Formula for Pooling","text":"<ul> <li>\\(O = \\frac {W - K}{S} + 1\\)<ul> <li>W: input height/width</li> <li>K: filter size = 2</li> <li>S: stride size = filter size, PyTorch defaults the stride to kernel filter size<ul> <li>If using PyTorch default stride, this will result in the formula \\(O = \\frac {W}{K}\\)</li> <li>By default, in our tutorials, we do this for simplicity.</li> </ul> </li> </ul> </li> </ul> <p>Define our simple 2 convolutional layer CNN</p> <pre><code>class CNNModel(nn.Module):\n    def __init__(self):\n        super(CNNModel, self).__init__()\n\n        # Convolution 1\n        self.cnn1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=5, stride=1, padding=2)\n        self.relu1 = nn.ReLU()\n\n        # Max pool 1\n        self.maxpool1 = nn.MaxPool2d(kernel_size=2)\n\n        # Convolution 2\n        self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=2)\n        self.relu2 = nn.ReLU()\n\n        # Max pool 2\n        self.maxpool2 = nn.MaxPool2d(kernel_size=2)\n\n        # Fully connected 1 (readout)\n        self.fc1 = nn.Linear(32 * 7 * 7, 10) \n\n    def forward(self, x):\n        # Convolution 1\n        out = self.cnn1(x)\n        out = self.relu1(out)\n\n        # Max pool 1\n        out = self.maxpool1(out)\n\n        # Convolution 2 \n        out = self.cnn2(out)\n        out = self.relu2(out)\n\n        # Max pool 2 \n        out = self.maxpool2(out)\n\n        # Resize\n        # Original size: (100, 32, 7, 7)\n        # out.size(0): 100\n        # New out size: (100, 32*7*7)\n        out = out.view(out.size(0), -1)\n\n        # Linear function (readout)\n        out = self.fc1(out)\n\n        return out\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#step-4-instantiate-model-class","title":"Step 4: Instantiate Model Class","text":"<p>Our model</p> <pre><code>model = CNNModel()\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#step-5-instantiate-loss-class","title":"Step 5: Instantiate Loss Class","text":"<ul> <li>Convolutional Neural Network: Cross Entropy Loss<ul> <li>Feedforward Neural Network: Cross Entropy Loss</li> <li>Logistic Regression: Cross Entropy Loss</li> <li>Linear Regression: MSE</li> </ul> </li> </ul> <p>Our cross entropy loss</p> <pre><code>criterion = nn.CrossEntropyLoss()\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#step-6-instantiate-optimizer-class","title":"Step 6: Instantiate Optimizer Class","text":"<ul> <li>Simplified equation<ul> <li>\\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\)<ul> <li>\\(\\theta\\): parameters</li> <li>\\(\\eta\\): learning rate (how fast we want to learn)</li> <li>\\(\\nabla_\\theta\\): parameters' gradients</li> </ul> </li> </ul> </li> <li>Even simplier equation<ul> <li><code>parameters = parameters - learning_rate * parameters_gradients</code></li> <li>At every iteration, we update our model's parameters</li> </ul> </li> </ul> <p>Optimizer</p> <pre><code>learning_rate = 0.01\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)  \n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#parameters-in-depth","title":"Parameters In-Depth","text":"<p>Print model's parameter</p> <pre><code>print(model.parameters())\n\nprint(len(list(model.parameters())))\n\n# Convolution 1: 16 Kernels\nprint(list(model.parameters())[0].size())\n\n# Convolution 1 Bias: 16 Kernels\nprint(list(model.parameters())[1].size())\n\n# Convolution 2: 32 Kernels with depth = 16\nprint(list(model.parameters())[2].size())\n\n# Convolution 2 Bias: 32 Kernels with depth = 16\nprint(list(model.parameters())[3].size())\n\n# Fully Connected Layer 1\nprint(list(model.parameters())[4].size())\n\n# Fully Connected Layer Bias\nprint(list(model.parameters())[5].size())\n</code></pre> <pre><code>&lt;generator object Module.parameters at 0x7f9864363c50&gt;\n6\ntorch.Size([16, 1, 5, 5])\ntorch.Size([16])\ntorch.Size([32, 16, 5, 5])\ntorch.Size([32])\ntorch.Size([10, 1568])\ntorch.Size([10])\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#step-7-train-model","title":"Step 7: Train Model","text":"<ul> <li>Process <ol> <li>Convert inputs to tensors with gradient accumulation abilities<ul> <li>CNN Input: (1, 28, 28) </li> <li>Feedforward NN Input: (1, 28*28)</li> </ul> </li> <li>Clear gradient buffets</li> <li>Get output given inputs </li> <li>Get loss</li> <li>Get gradients w.r.t. parameters</li> <li>Update parameters using gradients<ul> <li><code>parameters = parameters - learning_rate * parameters_gradients</code></li> </ul> </li> <li>REPEAT</li> </ol> </li> </ul> <p>Model training</p> <pre><code>iter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images\n        images = images.requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images\n                images = images.requires_grad_()\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 0.43324267864227295. Accuracy: 90\nIteration: 1000. Loss: 0.2511480152606964. Accuracy: 92\nIteration: 1500. Loss: 0.13431282341480255. Accuracy: 94\nIteration: 2000. Loss: 0.11173319816589355. Accuracy: 95\nIteration: 2500. Loss: 0.06409914791584015. Accuracy: 96\nIteration: 3000. Loss: 0.14377528429031372. Accuracy: 96\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#model-b","title":"Model B:","text":"<ul> <li>2 Convolutional Layers<ul> <li>Same Padding (same output size)</li> </ul> </li> <li>2 Average Pooling Layers</li> <li>1 Fully Connected Layer</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#steps_1","title":"Steps","text":"<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> </ul> <p>2 Conv + 2 Average Pool + 1 FC (Zero Padding, Same Padding)</p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass CNNModel(nn.Module):\n    def __init__(self):\n        super(CNNModel, self).__init__()\n\n        # Convolution 1\n        self.cnn1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=5, stride=1, padding=2)\n        self.relu1 = nn.ReLU()\n\n        # Average pool 1\n        self.avgpool1 = nn.AvgPool2d(kernel_size=2)\n\n        # Convolution 2\n        self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=2)\n        self.relu2 = nn.ReLU()\n\n        # Average pool 2\n        self.avgpool2 = nn.AvgPool2d(kernel_size=2)\n\n        # Fully connected 1 (readout)\n        self.fc1 = nn.Linear(32 * 7 * 7, 10) \n\n    def forward(self, x):\n        # Convolution 1\n        out = self.cnn1(x)\n        out = self.relu1(out)\n\n        # Average pool 1\n        out = self.avgpool1(out)\n\n        # Convolution 2 \n        out = self.cnn2(out)\n        out = self.relu2(out)\n\n        # Max pool 2 \n        out = self.avgpool2(out)\n\n        # Resize\n        # Original size: (100, 32, 7, 7)\n        # out.size(0): 100\n        # New out size: (100, 32*7*7)\n        out = out.view(out.size(0), -1)\n\n        # Linear function (readout)\n        out = self.fc1(out)\n\n        return out\n\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\n\nmodel = CNNModel()\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.01\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as tensors with gradient accumulation abilities\n        images = images.requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to tensors with gradient accumulation abilities\n                images = images.requires_grad_()\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 0.6850348711013794. Accuracy: 85\nIteration: 1000. Loss: 0.36549052596092224. Accuracy: 88\nIteration: 1500. Loss: 0.31540098786354065. Accuracy: 89\nIteration: 2000. Loss: 0.3522164225578308. Accuracy: 90\nIteration: 2500. Loss: 0.2680729925632477. Accuracy: 91\nIteration: 3000. Loss: 0.26440390944480896. Accuracy: 92\n</code></pre> <p>Comparison of accuracies</p> <p>It seems like average pooling test accuracy is less than the max pooling accuracy! Does this mean average pooling is better? This is not definitive and depends on a lot of factors including the model's architecture, seed (that affects random weight initialization) and more.</p>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#model-c","title":"Model C:","text":"<ul> <li>2 Convolutional Layers<ul> <li>Valid Padding (smaller output size)</li> </ul> </li> <li>2 Max Pooling Layers</li> <li>1 Fully Connected Layer</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#steps_2","title":"Steps","text":"<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> </ul> <p>2 Conv + 2 Max Pool + 1 FC (Valid Padding, No Padding)</p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass CNNModel(nn.Module):\n    def __init__(self):\n        super(CNNModel, self).__init__()\n\n        # Convolution 1\n        self.cnn1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=5, stride=1, padding=0)\n        self.relu1 = nn.ReLU()\n\n        # Max pool 1\n        self.maxpool1 = nn.MaxPool2d(kernel_size=2)\n\n        # Convolution 2\n        self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=0)\n        self.relu2 = nn.ReLU()\n\n        # Max pool 2\n        self.maxpool2 = nn.MaxPool2d(kernel_size=2)\n\n        # Fully connected 1 (readout)\n        self.fc1 = nn.Linear(32 * 4 * 4, 10) \n\n    def forward(self, x):\n        # Convolution 1\n        out = self.cnn1(x)\n        out = self.relu1(out)\n\n        # Max pool 1\n        out = self.maxpool1(out)\n\n        # Convolution 2 \n        out = self.cnn2(out)\n        out = self.relu2(out)\n\n        # Max pool 2 \n        out = self.maxpool2(out)\n\n        # Resize\n        # Original size: (100, 32, 7, 7)\n        # out.size(0): 100\n        # New out size: (100, 32*7*7)\n        out = out.view(out.size(0), -1)\n\n        # Linear function (readout)\n        out = self.fc1(out)\n\n        return out\n\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\n\nmodel = CNNModel()\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.01\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as tensors with gradient accumulation abilities\n        images = images.requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to tensors with gradient accumulation abilities\n                images = images.requires_grad_()\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 0.5153220295906067. Accuracy: 88\nIteration: 1000. Loss: 0.28784745931625366. Accuracy: 92\nIteration: 1500. Loss: 0.4086027443408966. Accuracy: 94\nIteration: 2000. Loss: 0.09390712529420853. Accuracy: 95\nIteration: 2500. Loss: 0.07138358801603317. Accuracy: 95\nIteration: 3000. Loss: 0.05396252125501633. Accuracy: 96\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#summary-of-results","title":"Summary of Results","text":"Model A Model B Model C Max Pooling Average Pooling Max Pooling Same Padding Same Padding Valid Padding 97.04% 93.59% 96.5% All Models INPUT \\(\\rightarrow\\) CONV \\(\\rightarrow\\) POOL \\(\\rightarrow\\) CONV \\(\\rightarrow\\) POOL \\(\\rightarrow\\) FC Convolution Kernel Size = 5 x 5 Convolution Kernel Stride = 1 Pooling Kernel Size = 2 x 2"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#general-deep-learning-notes-on-cnn-and-fnn","title":"General Deep Learning Notes on CNN and FNN","text":"<ul> <li>3 ways to expand a convolutional neural network<ul> <li>More convolutional layers </li> <li>Less aggressive downsampling<ul> <li>Smaller kernel size for pooling (gradually downsampling)</li> </ul> </li> <li>More fully connected layers </li> </ul> </li> <li>Cons<ul> <li>Need a larger dataset<ul> <li>Curse of dimensionality</li> </ul> </li> <li>Does not necessarily mean higher accuracy</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#3-building-a-convolutional-neural-network-with-pytorch-gpu","title":"3. Building a Convolutional Neural Network with PyTorch (GPU)","text":""},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#model-a_1","title":"Model A","text":"<p>GPU: 2 things must be on GPU - <code>model</code> - <code>tensors</code></p>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#steps_3","title":"Steps","text":"<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> </ul> <p>2 Conv + 2 Max Pooling + 1 FC (Same Padding, Zero Padding)</p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets \n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass CNNModel(nn.Module):\n    def __init__(self):\n        super(CNNModel, self).__init__()\n\n        # Convolution 1\n        self.cnn1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=5, stride=1, padding=0)\n        self.relu1 = nn.ReLU()\n\n        # Max pool 1\n        self.maxpool1 = nn.MaxPool2d(kernel_size=2)\n\n        # Convolution 2\n        self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=0)\n        self.relu2 = nn.ReLU()\n\n        # Max pool 2\n        self.maxpool2 = nn.MaxPool2d(kernel_size=2)\n\n        # Fully connected 1 (readout)\n        self.fc1 = nn.Linear(32 * 4 * 4, 10) \n\n    def forward(self, x):\n        # Convolution 1\n        out = self.cnn1(x)\n        out = self.relu1(out)\n\n        # Max pool 1\n        out = self.maxpool1(out)\n\n        # Convolution 2 \n        out = self.cnn2(out)\n        out = self.relu2(out)\n\n        # Max pool 2 \n        out = self.maxpool2(out)\n\n        # Resize\n        # Original size: (100, 32, 7, 7)\n        # out.size(0): 100\n        # New out size: (100, 32*7*7)\n        out = out.view(out.size(0), -1)\n\n        # Linear function (readout)\n        out = self.fc1(out)\n\n        return out\n\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\n\nmodel = CNNModel()\n\n#######################\n#  USE GPU FOR MODEL  #\n#######################\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.01\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n\n        #######################\n        #  USE GPU FOR MODEL  #\n        #######################\n        images = images.requires_grad_().to(device)\n        labels = labels.to(device)\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                #######################\n                #  USE GPU FOR MODEL  #\n                #######################\n                images = images.requires_grad_().to(device)\n                labels = labels.to(device)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                #######################\n                #  USE GPU FOR MODEL  #\n                #######################\n                # Total correct predictions\n                if torch.cuda.is_available():\n                    correct += (predicted.cpu() == labels.cpu()).sum()\n                else:\n                    correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 0.36831170320510864. Accuracy: 88\nIteration: 1000. Loss: 0.31790846586227417. Accuracy: 92\nIteration: 1500. Loss: 0.1510857343673706. Accuracy: 94\nIteration: 2000. Loss: 0.08368007838726044. Accuracy: 95\nIteration: 2500. Loss: 0.13419771194458008. Accuracy: 96\nIteration: 3000. Loss: 0.16750787198543549. Accuracy: 96\n</code></pre> <p>More Efficient Convolutions via Toeplitz Matrices</p> <p>This is beyond the scope of this particular lesson. But now that we understand how convolutions work, it is critical to know that it is quite an inefficient operation if we use for-loops to perform our 2D convolutions (5 x 5 convolution kernel size for example) on our 2D images (28 x 28 MNIST image for example).</p> <p>A more efficient implementation is in converting our convolution kernel into a doubly block circulant/Toeplitz matrix (special case Toeplitz matrix) and our image (input) into a vector. Then, we will do just one matrix operation using our doulby block Toeplitz matrix and our input vector.</p> <p>There will be a whole lesson dedicated to this operation released down the road. </p>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#summary","title":"Summary","text":"<p>We've learnt to...</p> <p>Success</p> <ul> <li> Transition from Feedforward Neural Network<ul> <li> Addition of Convolutional &amp; Pooling Layers before Linear Layers</li> </ul> </li> <li> One Convolutional Layer Basics</li> <li> One Pooling Layer Basics<ul> <li> Max pooling</li> <li> Average pooling</li> </ul> </li> <li> Padding</li> <li> Output Dimension Calculations and Examples<ul> <li> \\(O = \\frac {W - K + 2P}{S} + 1\\)</li> </ul> </li> <li> Convolutional Neural Networks<ul> <li> Model A: 2 Conv + 2 Max pool + 1 FC<ul> <li> Same Padding</li> </ul> </li> <li> Model B: 2 Conv + 2 Average pool + 1 FC<ul> <li> Same Padding</li> </ul> </li> <li> Model C: 2 Conv + 2 Max pool + 1 FC<ul> <li> Valid Padding</li> </ul> </li> </ul> </li> <li> Model Variation in Code<ul> <li> Modifying only step 3</li> </ul> </li> <li> Ways to Expand Model\u2019s Capacity<ul> <li> More convolutions</li> <li> Gradual pooling</li> <li> More fully connected layers</li> </ul> </li> <li> GPU Code<ul> <li> 2 things on GPU<ul> <li> model</li> <li> tensors with gradient accumulation abilities</li> </ul> </li> <li> Modifying only Step 4 &amp; Step 7</li> </ul> </li> <li> 7 Step Model Building Recap<ul> <li> Step 1: Load Dataset</li> <li> Step 2: Make Dataset Iterable</li> <li> Step 3: Create Model Class</li> <li> Step 4: Instantiate Model Class</li> <li> Step 5: Instantiate Loss Class</li> <li> Step 6: Instantiate Optimizer Class</li> <li> Step 7: Train Model</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/#citation","title":"Citation","text":"<p>If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.</p> <p></p>"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/","title":"Overcomplete Autoencoders with PyTorch","text":"<p>Run Jupyter Notebook</p> <p>You can run the code for this section in this jupyter notebook link.</p>"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#introduction","title":"Introduction","text":"<ul> <li>Recap!<ul> <li>An autoencoder's purpose is to learn an approximation of the identity function (mapping \\(x\\) to \\(\\hat x\\)).<ul> <li>Essentially we are trying to learn a function that can take our input \\(x\\) and recreate it \\(\\hat x\\).<ul> <li>Technically we can do an exact recreation of our in-sample input if we use a very wide and deep neural network.</li> </ul> </li> </ul> </li> </ul> </li> <li>In this particular tutorial, we will be covering denoising autoencoder through overcomplete encoders. <ul> <li>Essentially given noisy images, you can denoise and make them less noisy with this tutorial through overcomplete encoders.</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#fashion-mnist-dataset-exploration","title":"Fashion MNIST Dataset Exploration","text":""},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#imports","title":"Imports","text":"<pre><code>import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\nimport matplotlib.pyplot as plt\nimport numpy as np\n%matplotlib inline\nfigsize=(15, 6)\nplt.style.use('fivethirtyeight')\n</code></pre> Name Content Examples Size Link MD5 Checksum <code>train-images-idx3-ubyte.gz</code> training set images 60,000 26 MBytes Download <code>8d4fb7e6c68d591d4c3dfef9ec88bf0d</code> <code>train-labels-idx1-ubyte.gz</code> training set labels 60,000 29 KBytes Download <code>25c81989df183df01b3e8a0aad5dffbe</code> <code>t10k-images-idx3-ubyte.gz</code> test set images 10,000 4.3 MBytes Download <code>bef4ecab320f06d8554ea6380940ec79</code> <code>t10k-labels-idx1-ubyte.gz</code> test set labels 10,000 5.1 KBytes Download <code>bb300cfdad3c16e7a12a480ee83cd310</code>"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#load-dataset-step-1","title":"Load Dataset (Step 1)","text":"<ul> <li>Typically we've been leveraging on <code>dsets.MNIST()</code>, now we simply change to <code>dsets.FashionMNIST</code>!</li> </ul> <pre><code># Fashion-MNIST data loader\ntrain_dataset = dsets.FashionMNIST(root='./data',\n                                   train=True,\n                                   transform=transforms.ToTensor(),\n                                   download=True)\n\ntest_dataset = dsets.FashionMNIST(root='./data',\n                                  train=False,\n                                  transform=transforms.ToTensor())\n</code></pre> <pre><code>Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz\nDownloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz\nDownloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz\nDownloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz\nProcessing...\nDone!\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#load-data-loader-step-2","title":"Load Data Loader (Step 2)","text":"<pre><code># Batch size, iterations and epochs\nbatch_size = 100\nn_iters = 5000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#labels","title":"Labels","text":"<p>Each training and test example is assigned to one of the following labels:</p> Label Description 0 T-shirt/top 1 Trouser 2 Pullover 3 Dress 4 Coat 5 Sandal 6 Shirt 7 Sneaker 8 Bag 9 Ankle boot"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#sample-boot","title":"Sample: Boot","text":"<pre><code># Sample 0: boot\nsample_num = 0\nshow_img = train_dataset[sample_num][0].numpy().reshape(28, 28)\nlabel = train_dataset[sample_num][1]\nprint(f'Label {label}')\nplt.imshow(show_img, cmap='gray');\n</code></pre> <pre><code>Label 9\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#sample-shirt","title":"Sample: shirt","text":"<pre><code># Sample 1: shirt\nsample_num = 1\nshow_img = train_dataset[sample_num][0].numpy().reshape(28, 28)\nlabel = train_dataset[sample_num][1]\nprint(f'Label {label}')\nplt.imshow(show_img, cmap='gray');\n</code></pre> <pre><code>Label 0\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#sample-dress","title":"Sample: dress","text":"<pre><code># Sample 3: dress\nsample_num = 3\nshow_img = train_dataset[sample_num][0].numpy().reshape(28, 28)\nlabel = train_dataset[sample_num][1]\nprint(f'Label {label}')\nplt.imshow(show_img, cmap='gray');\n</code></pre> <pre><code>Label 3\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#maximumminimum-pixel-values","title":"Maximum/minimum pixel values","text":"<ul> <li>Pixel values range from 0 to 1</li> </ul> <pre><code>min_pixel_value = train_dataset[sample_num][0].min()\nmax_pixel_value = train_dataset[sample_num][0].max()\n\nprint(f'Minimum pixel value: {min_pixel_value}')\nprint(f'Maximum pixel value: {max_pixel_value}')\n</code></pre> <pre><code>Minimum pixel value: 0.0\nMaximum pixel value: 1.0\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#overcomplete-autoencoder","title":"Overcomplete Autoencoder","text":""},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#sigmoid-function","title":"Sigmoid Function","text":"<ul> <li>Sigmoid function was introduced earlier, where the function allows to bound our output from 0 to 1 inclusive given our input.</li> <li>This is introduced and clarified here as we would want this in our final layer of our overcomplete autoencoder as we want to bound out final output to the pixels' range of 0 and 1.</li> </ul> <pre><code># Sigmoid function has function bounded by min=0 and max=1\n# So this will be what we will be using for the final layer's function\nx = torch.arange(-10., 10., 0.1)\nplt.figure(figsize=figsize);\nplt.plot(x.numpy(), torch.sigmoid(x).numpy())\nplt.title('Sigmoid Function')\n</code></pre> <pre><code>Text(0.5, 1.0, 'Sigmoid Function')\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#steps","title":"Steps","text":"<ul> <li>These steps should be familiar by now! Our famous 7 steps. But I will be adding one more step here, Step 8 where we run our inference.<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> <li>Step 8: Model Inference</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#step-3-create-model-class","title":"Step 3: Create Model Class","text":"<pre><code># Model definition\nclass FullyConnectedAutoencoder(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super().__init__()\n        # Encoder: affine function\n        self.fc1 = nn.Linear(input_dim, hidden_dim)\n        # Decoder: affine function\n        self.fc2 = nn.Linear(hidden_dim, output_dim)\n\n    def forward(self, x):\n        # Encoder: affine function\n        out = self.fc1(x)\n        # Encoder: non-linear function\n        out = F.leaky_relu(out)\n\n        # Decoder: affine function\n        out = self.fc2(out)\n        # Decoder: non-linear function\n        out = torch.sigmoid(out)\n\n        return out\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#step-4-instantiate-model-class","title":"Step 4: Instantiate Model Class","text":"<pre><code># Dimensions for overcomplete (larger latent representation)\ninput_dim = 28*28\nhidden_dim = int(input_dim * 1.5)\noutput_dim = input_dim\n\n# Instantiate Fully-connected Autoencoder (FC-AE)\n# And assign to model object\nmodel = FullyConnectedAutoencoder(input_dim, hidden_dim, output_dim)\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#step-5-instantiate-loss-class","title":"Step 5: Instantiate Loss Class","text":"<pre><code># We want to minimize the per pixel reconstruction loss\n# So we've to use the mean squared error (MSE) loss\n# This is similar to our regression tasks' loss\ncriterion = nn.MSELoss()\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#step-6-instantiate-optimizer-class","title":"Step 6: Instantiate Optimizer Class","text":"<pre><code># Using basic Adam optimizer\nlearning_rate = 1e-3\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)  \n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#inspect-parameter-groups","title":"Inspect Parameter Groups","text":"<pre><code># Parameter inspection\nnum_params_group = len(list(model.parameters()))\nfor group_idx in range(num_params_group):\n    print(list(model.parameters())[group_idx].size())\n</code></pre> <pre><code>torch.Size([1176, 784])\ntorch.Size([1176])\ntorch.Size([784, 1176])\ntorch.Size([784])\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#step-7-train-model","title":"Step 7: Train Model","text":"<ul> <li>Take note there's a critical line of <code>dropout = nn.Dropout(0.5)</code> here which basically allows us to make noisy images from our original Fashion MNIST images.</li> <li>It basically drops out 50% of all pixels randomly.</li> </ul> <pre><code>idx = 0\n\n# Dropout for creating noisy images\n# by dropping out pixel with a 50% probability\ndropout = nn.Dropout(0.5)\n\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images with gradient accumulation capabilities\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Noisy images\n        noisy_images = dropout(torch.ones(images.shape)) * images\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output\n        outputs = model(noisy_images)\n\n        # Calculate Loss: MSE Loss based on pixel-to-pixel comparison\n        loss = criterion(outputs, images)\n\n        # Getting gradients w.r.t. parameters via backpropagation\n        loss.backward()\n\n        # Updating parameters via gradient descent\n        optimizer.step()\n\n        idx += 1\n\n        if idx % 500 == 0:\n            # Calculate MSE Test Loss\n            total_test_loss = 0\n            total_samples = 0\n\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Noisy images\n                noisy_images = dropout(torch.ones(images.shape)) * images\n\n                # Forward pass only to get logits/output\n                outputs = model(noisy_images.view(-1, 28*28))\n\n                # Test loss\n                test_loss = criterion(outputs, images.view(-1, 28*28))\n\n                # Total number of labels\n                total_samples += labels.size(0)\n\n                # Total test loss\n                total_test_loss += test_loss\n\n            mean_test_loss = total_test_loss / total_samples\n\n            # Print Loss\n            print(f'Iteration: {idx}. Average Test Loss: {mean_test_loss.item()}.')\n</code></pre> <pre><code>Iteration: 500. Average Test Loss: 0.0001664187147980556.\nIteration: 1000. Average Test Loss: 0.00014121478307060897.\nIteration: 1500. Average Test Loss: 0.0001341506722383201.\nIteration: 2000. Average Test Loss: 0.0001252180663868785.\nIteration: 2500. Average Test Loss: 0.00012206179235363379.\nIteration: 3000. Average Test Loss: 0.00011766648094635457.\nIteration: 3500. Average Test Loss: 0.00011584569438127801.\nIteration: 4000. Average Test Loss: 0.00011396891932236031.\nIteration: 4500. Average Test Loss: 0.00011224475019844249.\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#model-inference","title":"Model Inference","text":""},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#raw-sample-1","title":"Raw Sample 1","text":"<pre><code># Test sample: Raw\nsample_num = 10\nraw_img = test_dataset[sample_num][0]\nshow_img = raw_img.numpy().reshape(28, 28)\nlabel = test_dataset[sample_num][1]\nprint(f'Label {label}')\nplt.imshow(show_img, cmap='gray');\n</code></pre> <pre><code>Label 4\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#test-sample-1-noisy-to-denoised","title":"Test Sample 1: Noisy to Denoised","text":"<pre><code># Test sample: Noisy\nsample_num = 10\nraw_img = test_dataset[sample_num][0]\nnoisy_image = dropout(torch.ones(raw_img.shape)) * raw_img\nshow_img = noisy_image.numpy().reshape(28, 28)\nlabel = test_dataset[sample_num][1]\nprint(f'Label {label}')\nplt.imshow(show_img, cmap='gray');\n</code></pre> <pre><code>Label 4\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/#summary","title":"Summary","text":"<ul> <li>Introduction<ul> <li>Recap of Autoencoders</li> <li>Introduction of denoising autoencoders</li> </ul> </li> <li>Dataset Exploration<ul> <li>Fashion MNIST<ul> <li>10 classes</li> <li>Similar to MNIST but fashion images instead of digits</li> </ul> </li> </ul> </li> <li>8 Step Model Building Recap<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> <li>Step 8: Model Inference<ul> <li>Raw Sample 1</li> <li>Noisy to Denoised Sample 1</li> </ul> </li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/","title":"Feedforward Neural Network with PyTorch","text":"<p>Run Jupyter Notebook</p> <p>You can run the code for this section in this jupyter notebook link.</p>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#about-feedforward-neural-network","title":"About Feedforward Neural Network","text":""},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#logistic-regression-transition-to-neural-networks","title":"Logistic Regression Transition to Neural Networks","text":""},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#logistic-regression-review","title":"Logistic Regression Review","text":"<p>Define logistic regression model</p> <p>Import our relevant torch modules. <pre><code>import torch\nimport torch.nn as nn\n</code></pre></p> <p>Define our model class. <pre><code>class LogisticRegressionModel(nn.Module):\n    def __init__(self, input_dim, output_dim):\n        super(LogisticRegressionModel, self).__init__()\n        self.linear = nn.Linear(input_dim, output_dim)\n\n    def forward(self, x):\n        out = self.linear(x)\n        return out\n</code></pre></p> <p>Instantiate the logistic regression model. <pre><code>input_dim = 28*28\noutput_dim = 10\n\nmodel = LogisticRegressionModel(input_dim, output_dim)\n</code></pre></p> <p>When we inspect the model, we would have an input size of 784 (derived from 28 x 28) and output size of 10 (which is the number of classes we are classifying from 0 to 9). <pre><code>print(model)\n</code></pre></p> <pre><code>LogisticRegressionModel(\n  (linear): Linear(in_features=784, out_features=10, bias=True)\n)\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#logistic-regression-problems","title":"Logistic Regression Problems","text":"<ul> <li>Can represent linear functions well<ul> <li>\\(y = 2x + 3\\)</li> <li>\\(y = x_1 + x_2\\)</li> <li>\\(y = x_1 + 3x_2 + 4x_3\\)</li> </ul> </li> <li>Cannot represent non-linear functions<ul> <li>\\(y = 4x_1 + 2x_2^2 +3x_3^3\\)</li> <li>\\(y = x_1x_2\\)</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#introducing-a-non-linear-function","title":"Introducing a Non-linear Function","text":""},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#non-linear-function-in-depth","title":"Non-linear Function In-Depth","text":"<ul> <li>Function: takes a number &amp; perform mathematical operation</li> <li>Common Types of Non-linearity<ul> <li>ReLUs (Rectified Linear Units)      </li> <li>Sigmoid</li> <li>Tanh</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#sigmoid-logistic","title":"Sigmoid (Logistic)","text":"<ul> <li>\\(\\sigma(x) = \\frac{1}{1 + e^{-x}}\\)</li> <li>Input number \\(\\rightarrow\\) [0, 1]<ul> <li>Large negative number \\(\\rightarrow\\) 0</li> <li>Large positive number \\(\\rightarrow\\) 1</li> </ul> </li> <li>Cons: <ol> <li>Activation saturates at 0 or 1 with gradients \\(\\approx\\) 0<ul> <li>No signal to update weights \\(\\rightarrow\\) cannot learn</li> <li>Solution: Have to carefully initialize weights to prevent this</li> </ul> </li> <li>Outputs not centered around 0 <ul> <li>If output always positive \\(\\rightarrow\\) gradients always positive or negative \\(\\rightarrow\\) bad for gradient updates </li> </ul> </li> </ol> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#tanh","title":"Tanh","text":"<ul> <li>\\(\\tanh(x) = 2 \\sigma(2x) -1\\)<ul> <li>A scaled sigmoid function</li> </ul> </li> <li>Input number \\(\\rightarrow\\) [-1, 1]</li> <li>Cons: <ol> <li>Activation saturates at 0 or 1 with gradients \\(\\approx\\) 0<ul> <li>No signal to update weights \\(\\rightarrow\\) cannot learn</li> <li>Solution: Have to carefully initialize weights to prevent this</li> </ul> </li> </ol> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#relus","title":"ReLUs","text":"<ul> <li>\\(f(x) = \\max(0, x)\\)</li> <li>Pros:<ol> <li>Accelerates convergence \\(\\rightarrow\\) train faster</li> <li>Less computationally expensive operation compared to Sigmoid/Tanh exponentials</li> </ol> </li> <li>Cons:<ol> <li>Many ReLU units \"die\" \\(\\rightarrow\\) gradients = 0 forever<ul> <li>Solution: careful learning rate choice</li> </ul> </li> </ol> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#building-a-feedforward-neural-network-with-pytorch","title":"Building a Feedforward Neural Network with PyTorch","text":""},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#model-a-1-hidden-layer-feedforward-neural-network-sigmoid-activation","title":"Model A: 1 Hidden Layer Feedforward Neural Network (Sigmoid Activation)","text":""},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#steps","title":"Steps","text":"<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#step-1-loading-mnist-train-dataset","title":"Step 1: Loading MNIST Train Dataset","text":"<p>Images from 1 to 9</p> <p>Similar to what we did in logistic regression, we will be using the same MNIST dataset where we load our training and testing datasets.</p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n</code></pre> <pre><code>train_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#step-2-make-dataset-iterable","title":"Step 2: Make Dataset Iterable","text":"<p>Batch sizes and iterations</p> <p>Because we have 60000 training samples (images), we need to split them up to small groups (batches) and pass these batches of samples to our feedforward neural network subsesquently.</p> <p>There are a few reasons why we split them into batches. Passing your whole dataset as a single batch would:</p> <p>(1) require a lot of RAM/VRAM on your CPU/GPU and this might result in Out-of-Memory (OOM) errors. </p> <p>(2) cause unstable training if you just use all the errors accumulated in 60,000 images to update the model rather than gradually update the model. In layman terms, imagine you accumulated errors for a student taking an exam with 60,000 questions and punish the student all at the same time. It is much harder for the student to learn compared to letting the student learn it made mistakes and did well in smaller batches of questions like mini-tests!</p> <p>If we have 60,000 images and we want a batch size of 100, then we would have 600 iterations where each iteration involves passing 600 images to the model and getting their respective predictions. </p> <pre><code>60000 / 100\n</code></pre> <pre><code>600.0\n</code></pre> <p>Epochs</p> <p>An epoch means that you have successfully passed the whole training set, 60,000 images, to the model. Continuing our example above, an epoch consists of 600 iterations.</p> <p>If we want to go through the whole dataset 5 times (5 epochs) for the model to learn, then we need 3000 iterations (600 x 5). </p> <pre><code>600 * 5\n</code></pre> <pre><code>3000.0\n</code></pre> <p>Bringing batch size, iterations and epochs together</p> <p>As we have gone through above, we want to have 5 epochs, where each epoch would have 600 iterations and each iteration has a batch size of 100.</p> <p>Because we want 5 epochs, we need a total of 3000 iterations.</p> <pre><code>batch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#step-3-create-model-class","title":"Step 3: Create Model Class","text":"<p>Creating our feedforward neural network</p> <p>Compared to logistic regression with only a single linear layer, we know for an FNN we need an additional linear layer and non-linear layer. </p> <p>This translates to just 4 more lines of code! </p> <pre><code>class FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n\n        # Non-linearity\n        self.sigmoid = nn.Sigmoid()\n\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function  # LINEAR\n        out = self.fc1(x)\n\n        # Non-linearity  # NON-LINEAR\n        out = self.sigmoid(out)\n\n        # Linear function (readout)  # LINEAR\n        out = self.fc2(out)\n        return out\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#step-4-instantiate-model-class","title":"Step 4: Instantiate Model Class","text":"<ul> <li>Input dimension: 784 <ul> <li>Size of image</li> <li>\\(28 \\times 28 = 784\\)</li> </ul> </li> <li>Output dimension: 10<ul> <li>0, 1, 2, 3, 4, 5, 6, 7, 8, 9</li> </ul> </li> <li>Hidden dimension: 100<ul> <li>Can be any number</li> <li>Similar term<ul> <li>Number of neurons</li> <li>Number of non-linear activation functions</li> </ul> </li> </ul> </li> </ul> <p>Instantiating our model class</p> <p>Our input size is determined by the size of the image (numbers ranging from 0 to 9) which has a width of 28 pixels and a height of 28 pixels. Hence the size of our input is 784 (28 x 28).</p> <p>Our output size is what we are trying to predict. When we pass an image to our model, it will try to predict if it's 0, 1, 2, 3, 4, 5, 6, 7, 8, or 9. That is a total of 10 classes, hence we have an output size of 10.</p> <p>Now the tricky part is in determining our hidden layer size, that is the size of our first linear layer prior to the non-linear layer. This can be any number, a larger number implies a bigger model with more parameters. Intuitively we think a bigger model equates to a better model, but a bigger model requires more training samples to learn and converge to a good model (also called curse of dimensionality). Hence, it is wise to pick the model size for the problem at hand. Because it is a simple problem of recognizing digits, we typically would not need a big model to achieve state-of-the-art results.</p> <p>On the flipside, too small of a hidden size would mean there would be insufficient model capacity to predict competently. In layman terms, too small of a capacity implies a smaller brain capacity so no matter how many training samples you give it, it has a maximum capacity in terms of its predictive power. </p> <pre><code>input_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#step-5-instantiate-loss-class","title":"Step 5: Instantiate Loss Class","text":"<ul> <li>Feedforward Neural Network: Cross Entropy Loss<ul> <li>Logistic Regression: Cross Entropy Loss</li> <li>Linear Regression: MSE</li> </ul> </li> </ul> <p>Loss class</p> <p>This is exactly the same as what we did in logistic regression. Because we are going through a classification problem, cross entropy function is required to compute the loss between our softmax outputs and our binary labels.</p> <pre><code>criterion = nn.CrossEntropyLoss()\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#step-6-instantiate-optimizer-class","title":"Step 6: Instantiate Optimizer Class","text":"<ul> <li>Simplified equation<ul> <li>\\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\)<ul> <li>\\(\\theta\\): parameters (our tensors with gradient accumulation capabilities)</li> <li>\\(\\eta\\): learning rate (how fast we want to learn)</li> <li>\\(\\nabla_\\theta\\): parameters' gradients</li> </ul> </li> </ul> </li> <li>Even simplier equation<ul> <li><code>parameters = parameters - learning_rate * parameters_gradients</code></li> <li>At every iteration, we update our model's parameters</li> </ul> </li> </ul> <p>Optimizer class</p> <p>Learning rate determines how fast the algorithm learns. Too small and the algorithm learns too slowly, too large and the algorithm learns too fast resulting in instabilities.</p> <p>Intuitively, we would think a larger learning rate would be better because we learn faster. But that's not true. Imagine we pass 10 images to a human to learn how to recognize whether the image is a hot dog or not, and it got half right and half wrong. </p> <p>A well defined learning rate (neither too small or large) is equivalent to rewarding the human with a sweet for getting the first half right, and punishing the other half the human got wrong with a smack on the palm. </p> <p>A large learning rate would be equivalent to feeding a thousand sweets to the human and smacking a thousand times on the human's palm. This would lead in a very unstable learning environment. Similarly, we will observe that the algorithm's convergence path will be extremely unstable if you use a large learning rate without reducing it subsequently. </p> <p>We are using an optimization algorithm called Stochastic Gradient Descent (SGD) which is essentially what we covered above on calculating the parameters' gradients multiplied by the learning rate then using it to update our parameters gradually. There's an in-depth analysis of various optimization algorithms on top of SGD in another section.</p> <pre><code>learning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)  \n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#parameters-in-depth","title":"Parameters In-Depth","text":"<p>Linear layers' parameters</p> <p>In a simple linear layer it's \\(Y = AX + B\\), and our parameters are \\(A\\) and bias \\(B\\). </p> <p>Hence, each linear layer would have 2 groups of parameters  \\(A\\) and \\(B\\). It is critical to take note that our non-linear layers have no parameters to update. They are merely mathematical functions performed on \\(Y\\), the output of our linear layers.</p> <p>This would return a Python generator object, so you need to call list on the generator object to access anything meaningful. <pre><code>print(model.parameters())\n</code></pre></p> <p>Here we call list on the generator object and getting the length of the list. This would return 4 because we've 2 linear layers, and each layer has 2 groups of parameters \\(A\\) and \\(b\\).</p> <pre><code>print(len(list(model.parameters())))\n</code></pre> <p>Our first linear layer parameters, \\(A_1\\), would be of size 100 x 784. This is because we've an input size of 784 (28 x 28) and a hidden size of 100. <pre><code># FC 1 Parameters \nprint(list(model.parameters())[0].size())\n</code></pre></p> <p>Our first linear layer bias parameters, \\(B_1\\), would be of size 100 which is our hidden size. <pre><code># FC 1 Bias Parameters\nprint(list(model.parameters())[1].size())\n</code></pre></p> <p>Our second linear layer is our readout layer, where the parameters \\(A_2\\) would be of size 10 x 100. This is because our output size is 10 and hidden size is 100.</p> <pre><code># FC 2 Parameters\nprint(list(model.parameters())[2].size())\n</code></pre> <p>Likewise our readout layer's bias \\(B_1\\) would just be 10, the size of our output.</p> <pre><code># FC 2 Bias Parameters\nprint(list(model.parameters())[3].size())\n</code></pre> <p>The diagram below shows the interaction amongst our input \\(X\\) and our linear layers' parameters \\(A_1\\), \\(B_1\\), \\(A_2\\), and \\(B_2\\) to reach to the final size of 10 x 1.</p> <p>If you're still unfamiliar with matrix product, go ahead and review the previous quick lesson where we covered it in logistic regression.</p> <pre><code>&lt;generator object Module.parameters at 0x7f1d530fa678&gt;\n4\ntorch.Size([100, 784])\ntorch.Size([100])\ntorch.Size([10, 100])\ntorch.Size([10])\n</code></pre> <p></p>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#step-7-train-model","title":"Step 7: Train Model","text":"<ul> <li>Process <ol> <li>Convert inputs to tensors with gradient accumulation capabilities</li> <li>Clear gradient buffers</li> <li>Get output given inputs </li> <li>Get loss</li> <li>Get gradients w.r.t. parameters</li> <li>Update parameters using gradients<ul> <li><code>parameters = parameters - learning_rate * parameters_gradients</code></li> </ul> </li> <li>REPEAT</li> </ol> </li> </ul> <p>7-step training process</p> <pre><code>iter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images with gradient accumulation capabilities\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images with gradient accumulation capabilities\n                images = images.view(-1, 28*28).requires_grad_()\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 0.6457265615463257. Accuracy: 85\nIteration: 1000. Loss: 0.39627206325531006. Accuracy: 89\nIteration: 1500. Loss: 0.2831554412841797. Accuracy: 90\nIteration: 2000. Loss: 0.4409525394439697. Accuracy: 91\nIteration: 2500. Loss: 0.2397005707025528. Accuracy: 91\nIteration: 3000. Loss: 0.3160165846347809. Accuracy: 91\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#model-b-1-hidden-layer-feedforward-neural-network-tanh-activation","title":"Model B: 1 Hidden Layer Feedforward Neural Network (Tanh Activation)","text":""},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#steps_1","title":"Steps","text":"<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> </ul> <p>1-layer FNN with Tanh Activation</p> <p>The only difference here compared to previously is that we are using Tanh activation instead of Sigmoid activation. This affects step 3. <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Non-linearity\n        self.tanh = nn.Tanh()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.tanh(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images with gradient accumulation capabilities\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images with gradient accumulation capabilities\n                images = images.view(-1, 28*28).requires_grad_()\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre></p> <pre><code>Iteration: 500. Loss: 0.4128190577030182. Accuracy: 91\nIteration: 1000. Loss: 0.14497484266757965. Accuracy: 92\nIteration: 1500. Loss: 0.272532194852829. Accuracy: 93\nIteration: 2000. Loss: 0.2758277952671051. Accuracy: 94\nIteration: 2500. Loss: 0.1603182554244995. Accuracy: 94\nIteration: 3000. Loss: 0.08848697692155838. Accuracy: 95\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#model-c-1-hidden-layer-feedforward-neural-network-relu-activation","title":"Model C: 1 Hidden Layer Feedforward Neural Network (ReLU Activation)","text":""},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#steps_2","title":"Steps","text":"<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> </ul> <p>1-layer FNN with ReLU Activation</p> <p>The only difference again is in using ReLU activation and it affects step 3. <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Non-linearity\n        self.relu = nn.ReLU()\n        # Linear function (readout)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function\n        out = self.fc1(x)\n        # Non-linearity\n        out = self.relu(out)\n        # Linear function (readout)\n        out = self.fc2(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images with gradient accumulation capabilities\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images with gradient accumulation capabilities\n                images = images.view(-1, 28*28).requires_grad_()\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre></p> <pre><code>Iteration: 500. Loss: 0.3179700970649719. Accuracy: 91\nIteration: 1000. Loss: 0.17288273572921753. Accuracy: 93\nIteration: 1500. Loss: 0.16829034686088562. Accuracy: 94\nIteration: 2000. Loss: 0.25494423508644104. Accuracy: 94\nIteration: 2500. Loss: 0.16818439960479736. Accuracy: 95\nIteration: 3000. Loss: 0.11110792309045792. Accuracy: 95\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#model-d-2-hidden-layer-feedforward-neural-network-relu-activation","title":"Model D: 2 Hidden Layer Feedforward Neural Network (ReLU Activation)","text":""},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#steps_3","title":"Steps","text":"<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> </ul> <p>2-layer FNN with ReLU Activation</p> <p>This is a bigger difference that increases your model's capacity by adding another linear layer and non-linear layer which affects step 3. <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function 1: 784 --&gt; 100\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Non-linearity 1\n        self.relu1 = nn.ReLU()\n\n        # Linear function 2: 100 --&gt; 100\n        self.fc2 = nn.Linear(hidden_dim, hidden_dim)\n        # Non-linearity 2\n        self.relu2 = nn.ReLU()\n\n        # Linear function 3 (readout): 100 --&gt; 10\n        self.fc3 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function 1\n        out = self.fc1(x)\n        # Non-linearity 1\n        out = self.relu1(out)\n\n        # Linear function 2\n        out = self.fc2(out)\n        # Non-linearity 2\n        out = self.relu2(out)\n\n        # Linear function 3 (readout)\n        out = self.fc3(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images with gradient accumulation capabilities\n        images = images.view(-1, 28*28).requires_grad_()\n        labels = labels\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images with gradient accumulation capabilities\n                images = images.view(-1, 28*28).requires_grad_()\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre></p> <pre><code>Iteration: 500. Loss: 0.2995373010635376. Accuracy: 91\nIteration: 1000. Loss: 0.3924565613269806. Accuracy: 93\nIteration: 1500. Loss: 0.1283276081085205. Accuracy: 94\nIteration: 2000. Loss: 0.10905527323484421. Accuracy: 95\nIteration: 2500. Loss: 0.11943754553794861. Accuracy: 96\nIteration: 3000. Loss: 0.15632082521915436. Accuracy: 96\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#model-e-3-hidden-layer-feedforward-neural-network-relu-activation","title":"Model E: 3 Hidden Layer Feedforward Neural Network (ReLU Activation)","text":""},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#steps_4","title":"Steps","text":"<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> </ul> <p>3-layer FNN with ReLU Activation</p> <p>Let's add one more layer! Bigger model capacity. But will it be better? Remember what we talked about on curse of dimensionality?</p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function 1: 784 --&gt; 100\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Non-linearity 1\n        self.relu1 = nn.ReLU()\n\n        # Linear function 2: 100 --&gt; 100\n        self.fc2 = nn.Linear(hidden_dim, hidden_dim)\n        # Non-linearity 2\n        self.relu2 = nn.ReLU()\n\n        # Linear function 3: 100 --&gt; 100\n        self.fc3 = nn.Linear(hidden_dim, hidden_dim)\n        # Non-linearity 3\n        self.relu3 = nn.ReLU()\n\n        # Linear function 4 (readout): 100 --&gt; 10\n        self.fc4 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function 1\n        out = self.fc1(x)\n        # Non-linearity 1\n        out = self.relu1(out)\n\n        # Linear function 2\n        out = self.fc2(out)\n        # Non-linearity 2\n        out = self.relu2(out)\n\n        # Linear function 2\n        out = self.fc3(out)\n        # Non-linearity 2\n        out = self.relu3(out)\n\n        # Linear function 4 (readout)\n        out = self.fc4(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images with gradient accumulation capabilities\n        images = images.view(-1, 28*28).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images with gradient accumulation capabilities\n                images = images.view(-1, 28*28).requires_grad_()\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 0.33234935998916626. Accuracy: 89\nIteration: 1000. Loss: 0.3098006248474121. Accuracy: 94\nIteration: 1500. Loss: 0.12461677193641663. Accuracy: 95\nIteration: 2000. Loss: 0.14346086978912354. Accuracy: 96\nIteration: 2500. Loss: 0.03763459622859955. Accuracy: 96\nIteration: 3000. Loss: 0.1397182047367096. Accuracy: 97\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#general-comments-on-fnns","title":"General Comments on FNNs","text":"<ul> <li>2 ways to expand a neural network<ul> <li>More non-linear activation units (neurons)</li> <li>More hidden layers </li> </ul> </li> <li>Cons<ul> <li>Need a larger dataset<ul> <li>Curse of dimensionality</li> </ul> </li> <li>Does not necessarily mean higher accuracy</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#3-building-a-feedforward-neural-network-with-pytorch-gpu","title":"3. Building a Feedforward Neural Network with PyTorch (GPU)","text":"<p>GPU: 2 things must be on GPU - <code>model</code> - <code>tensors</code></p>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#steps_5","title":"Steps","text":"<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> </ul> <p>3-layer FNN with ReLU Activation on GPU</p> <p>Only step 4 and 7 of the CPU code will be affected and it's a simple change.</p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass FeedforwardNeuralNetModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(FeedforwardNeuralNetModel, self).__init__()\n        # Linear function 1: 784 --&gt; 100\n        self.fc1 = nn.Linear(input_dim, hidden_dim) \n        # Non-linearity 1\n        self.relu1 = nn.ReLU()\n\n        # Linear function 2: 100 --&gt; 100\n        self.fc2 = nn.Linear(hidden_dim, hidden_dim)\n        # Non-linearity 2\n        self.relu2 = nn.ReLU()\n\n        # Linear function 3: 100 --&gt; 100\n        self.fc3 = nn.Linear(hidden_dim, hidden_dim)\n        # Non-linearity 3\n        self.relu3 = nn.ReLU()\n\n        # Linear function 4 (readout): 100 --&gt; 10\n        self.fc4 = nn.Linear(hidden_dim, output_dim)  \n\n    def forward(self, x):\n        # Linear function 1\n        out = self.fc1(x)\n        # Non-linearity 1\n        out = self.relu1(out)\n\n        # Linear function 2\n        out = self.fc2(out)\n        # Non-linearity 2\n        out = self.relu2(out)\n\n        # Linear function 2\n        out = self.fc3(out)\n        # Non-linearity 2\n        out = self.relu3(out)\n\n        # Linear function 4 (readout)\n        out = self.fc4(out)\n        return out\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\nhidden_dim = 100\noutput_dim = 10\n\nmodel = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)\n\n#######################\n#  USE GPU FOR MODEL  #\n#######################\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n\n        #######################\n        #  USE GPU FOR MODEL  #\n        #######################\n        images = images.view(-1, 28*28).requires_grad_().to(device)\n        labels = labels.to(device)\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                #######################\n                #  USE GPU FOR MODEL  #\n                #######################\n                images = images.view(-1, 28*28).requires_grad_().to(device)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                #######################\n                #  USE GPU FOR MODEL  #\n                #######################\n                # Total correct predictions\n                if torch.cuda.is_available():\n                    correct += (predicted.cpu() == labels.cpu()).sum()\n                else:\n                    correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 0.3877025246620178. Accuracy: 90\nIteration: 1000. Loss: 0.1337055265903473. Accuracy: 93\nIteration: 1500. Loss: 0.2038637101650238. Accuracy: 95\nIteration: 2000. Loss: 0.17892278730869293. Accuracy: 95\nIteration: 2500. Loss: 0.14455552399158478. Accuracy: 96\nIteration: 3000. Loss: 0.024540524929761887. Accuracy: 96\n</code></pre> <p>Alternative Term of Neural Network</p> <p>The alternative term is Universal Function Approximator. This is because ultimately we are trying to find a function that maps our input, \\(X\\), to our output, \\(y\\). </p>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#summary","title":"Summary","text":"<p>We've learnt to...</p> <p>Success</p> <ul> <li> Logistic Regression Problems for Non-Linear Functions Representation<ul> <li> Cannot represent non-linear functions<ul> <li> $ y = 4x_1 + 2x_2^2 +3x_3^3 $</li> <li> $ y = x_1x_2$</li> </ul> </li> </ul> </li> <li> Introduced Non-Linearity to Logistic Regression to form a Neural Network</li> <li> Types of Non-Linearity<ul> <li> Sigmoid</li> <li> Tanh</li> <li> ReLU</li> </ul> </li> <li> Feedforward Neural Network Models<ul> <li> Model A: 1 hidden layer (sigmoid activation)</li> <li> Model B: 1 hidden layer (tanh activation)</li> <li> Model C: 1 hidden layer (ReLU activation)</li> <li> Model D: 2 hidden layers (ReLU activation)</li> <li> Model E: 3 hidden layers (ReLU activation)</li> </ul> </li> <li> Models Variation in Code<ul> <li> Modifying only step 3</li> </ul> </li> <li> Ways to Expand Model\u2019s Capacity<ul> <li> More non-linear activation units (neurons)</li> <li> More hidden layers</li> </ul> </li> <li> Cons of Expanding Capacity<ul> <li> Need more data</li> <li> Does not necessarily mean higher accuracy</li> </ul> </li> <li> GPU Code<ul> <li> 2 things on GPU<ul> <li> model</li> <li> tensors with gradient accumulation capabilities</li> </ul> </li> <li> Modifying only Step 4 &amp; Step 7</li> </ul> </li> <li> 7 Step Model Building Recap<ul> <li> Step 1: Load Dataset</li> <li> Step 2: Make Dataset Iterable</li> <li> Step 3: Create Model Class</li> <li> Step 4: Instantiate Model Class</li> <li> Step 5: Instantiate Loss Class</li> <li> Step 6: Instantiate Optimizer Class</li> <li> Step 7: Train Model</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/#citation","title":"Citation","text":"<p>If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.</p> <p></p>"},{"location":"deep_learning/practical_pytorch/pytorch_gradients/","title":"Gradients with PyTorch","text":"<p>Run Jupyter Notebook</p> <p>You can run the code for this section in this jupyter notebook link.</p>"},{"location":"deep_learning/practical_pytorch/pytorch_gradients/#tensors-with-gradients","title":"Tensors with Gradients","text":""},{"location":"deep_learning/practical_pytorch/pytorch_gradients/#creating-tensors-with-gradients","title":"Creating Tensors with Gradients","text":"<ul> <li>Allows accumulation of gradients</li> </ul> <p>Method 1: Create tensor with gradients</p> <p>It is very similar to creating a tensor, all you need to do is to add an additional argument.</p> <pre><code>import torch\n</code></pre> <pre><code>a = torch.ones((2, 2), requires_grad=True)\na\n</code></pre> <pre><code>tensor([[ 1.,  1.],\n        [ 1.,  1.]])\n</code></pre> <p>Check if tensor requires gradients</p> <p>This should return True otherwise you've not done it right. <pre><code>a.requires_grad\n</code></pre></p> <pre><code>True\n</code></pre> <p>Method 2: Create tensor with gradients</p> <p>This allows you to create a tensor as usual then an additional line to allow it to accumulate gradients.</p> <pre><code># Normal way of creating gradients\na = torch.ones((2, 2))\n\n# Requires gradient\na.requires_grad_()\n\n# Check if requires gradient\na.requires_grad\n</code></pre> <pre><code>True\n</code></pre> <p>A tensor without gradients just for comparison</p> <p>If you do not do either of the methods above, you'll realize you will get False for checking for gradients. <pre><code># Not a variable\nno_gradient = torch.ones(2, 2)\n</code></pre></p> <pre><code>no_gradient.requires_grad\n</code></pre> <pre><code>False\n</code></pre> <p>Tensor with gradients addition operation</p> <pre><code># Behaves similarly to tensors\nb = torch.ones((2, 2), requires_grad=True)\nprint(a + b)\nprint(torch.add(a, b))\n</code></pre> <pre><code>tensor([[ 2.,  2.],\n        [ 2.,  2.]])\n\ntensor([[ 2.,  2.],\n        [ 2.,  2.]])\n</code></pre> <p>Tensor with gradients multiplication operation</p> <p>As usual, the operations we learnt previously for tensors apply for tensors with gradients. Feel free to try divisions, mean or standard deviation! <pre><code>print(a * b)\nprint(torch.mul(a, b))\n</code></pre></p> <pre><code>tensor([[ 1.,  1.],\n        [ 1.,  1.]])\ntensor([[ 1.,  1.],\n        [ 1.,  1.]])\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_gradients/#manually-and-automatically-calculating-gradients","title":"Manually and Automatically Calculating Gradients","text":"<p>What exactly is <code>requires_grad</code>? - Allows calculation of gradients w.r.t. the tensor that all allows gradients accumulation</p> \\[y_i = 5(x_i+1)^2\\] <p>Create tensor of size 2x1 filled with 1's that requires gradient</p> <pre><code>x = torch.ones(2, requires_grad=True)\nx\n</code></pre> <pre><code>tensor([ 1.,  1.])\n</code></pre> <p>Simple linear equation with x tensor created</p> \\[y_i\\bigr\\rvert_{x_i=1} = 5(1 + 1)^2 = 5(2)^2 = 5(4) = 20\\] <p>We should get a value of 20 by replicating this simple equation </p> <pre><code>y = 5 * (x + 1) ** 2\ny\n</code></pre> <pre><code>tensor([ 20.,  20.])\n</code></pre> <p>Simple equation with y tensor</p> <p>Backward should be called only on a scalar (i.e. 1-element tensor) or with gradient w.r.t. the variable</p> <p>Let's reduce y to a scalar then...</p> \\[o = \\frac{1}{2}\\sum_i y_i\\] <p>As you can see above, we've a tensor filled with 20's, so average them would return 20</p> <pre><code>o = (1/2) * torch.sum(y)\no\n</code></pre> <pre><code>tensor(20.)\n</code></pre> <p>Calculating first derivative</p> <p> Recap <code>y</code> equation: \\(y_i = 5(x_i+1)^2\\) </p> <p> Recap <code>o</code> equation: \\(o = \\frac{1}{2}\\sum_i y_i\\) </p> <p> Substitute <code>y</code> into <code>o</code> equation: \\(o = \\frac{1}{2} \\sum_i 5(x_i+1)^2\\) </p> \\[\\frac{\\partial o}{\\partial x_i} = \\frac{1}{2}[10(x_i+1)]\\] \\[\\frac{\\partial o}{\\partial x_i}\\bigr\\rvert_{x_i=1} = \\frac{1}{2}[10(1 + 1)] = \\frac{10}{2}(2) = 10\\] <p>We should expect to get 10, and it's so simple to do this with PyTorch with the following line...</p> <p>Get first derivative: <pre><code>o.backward()\n</code></pre></p> <p>Print out first derivative:</p> <pre><code>x.grad\n</code></pre> <pre><code>tensor([ 10.,  10.])\n</code></pre> <p>If x requires gradient and you create new objects with it, you get all gradients</p> <pre><code>print(x.requires_grad)\nprint(y.requires_grad)\nprint(o.requires_grad)\n</code></pre> <pre><code>True\nTrue\nTrue\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_gradients/#summary","title":"Summary","text":"<p>We've learnt to...</p> <p>Success</p> <ul> <li> Tensor with Gradients<ul> <li> Wraps a tensor for gradient accumulation</li> </ul> </li> <li> Gradients<ul> <li> Define original equation</li> <li> Substitute equation with <code>x</code> values</li> <li> Reduce to scalar output, <code>o</code> through <code>mean</code></li> <li> Calculate gradients with <code>o.backward()</code></li> <li> Then access gradients of the <code>x</code> tensor with <code>requires_grad</code> through <code>x.grad</code></li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_gradients/#citation","title":"Citation","text":"<p>If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.</p> <p></p>"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/","title":"Linear Regression with PyTorch","text":"<p>Run Jupyter Notebook</p> <p>You can run the code for this section in this jupyter notebook link.</p>"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#about-linear-regression","title":"About Linear Regression","text":""},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#simple-linear-regression-basics","title":"Simple Linear Regression Basics","text":"<ul> <li>Allows us to understand relationship between two continuous variables</li> <li>Example<ul> <li>x: independent variable<ul> <li>weight</li> </ul> </li> <li>y: dependent variable<ul> <li>height</li> </ul> </li> </ul> </li> <li>\\(y = \\alpha x + \\beta\\)</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#example-of-simple-linear-regression","title":"Example of simple linear regression","text":"<p>Create plot for simple linear regression</p> <p>Take note that this code is not important at all. It simply creates random data points and does a simple best-fit line to best approximate the underlying function if one even exists.</p> <pre><code>import numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\n# Creates 50 random x and y numbers\nnp.random.seed(1)\nn = 50\nx = np.random.randn(n)\ny = x * np.random.randn(n)\n\n# Makes the dots colorful\ncolors = np.random.rand(n)\n\n# Plots best-fit line via polyfit\nplt.plot(np.unique(x), np.poly1d(np.polyfit(x, y, 1))(np.unique(x)))\n\n# Plots the random x and y data points we created\n# Interestingly, alpha makes it more aesthetically pleasing\nplt.scatter(x, y, c=colors, alpha=0.5)\nplt.show()\n</code></pre> <p></p>"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#aim-of-linear-regression","title":"Aim of Linear Regression","text":"<ul> <li>Minimize the distance between the points and the line (\\(y = \\alpha x + \\beta\\))</li> <li>Adjusting<ul> <li>Coefficient: \\(\\alpha\\)</li> <li>Bias/intercept: \\(\\beta\\)</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#building-a-linear-regression-model-with-pytorch","title":"Building a Linear Regression Model with PyTorch","text":""},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#example","title":"Example","text":"<ul> <li>Coefficient: \\(\\alpha = 2\\)</li> <li>Bias/intercept: \\(\\beta = 1\\)</li> <li>Equation: \\(y = 2x + 1\\)</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#building-a-toy-dataset","title":"Building a Toy Dataset","text":"<p>Create a list of values from 0 to 11</p> <pre><code>x_values = [i for i in range(11)]\n</code></pre> <pre><code>x_values\n</code></pre> <pre><code>[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n</code></pre> <p>Convert list of numbers to numpy array</p> <pre><code># Convert to numpy\nx_train = np.array(x_values, dtype=np.float32)\nx_train.shape\n</code></pre> <pre><code>(11,)\n</code></pre> <p>Convert to 2-dimensional array</p> <p>If you don't this you will get an error stating you need 2D. Simply just reshape accordingly if you ever face such errors down the road. <pre><code># IMPORTANT: 2D required\nx_train = x_train.reshape(-1, 1)\nx_train.shape\n</code></pre></p> <pre><code>(11, 1)\n</code></pre> <p>Create list of y values</p> <p>We want y values for every x value we have above. </p> <p>\\(y = 2x + 1\\)</p> <pre><code>y_values = [2*i + 1 for i in x_values]\n</code></pre> <pre><code>y_values\n</code></pre> <pre><code>[1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21]\n</code></pre> <p>Alternative to create list of y values</p> <p>If you're weak in list iterators, this might be an easier alternative. <pre><code># In case you're weak in list iterators...\ny_values = []\nfor i in x_values:\n    result = 2*i + 1\n    y_values.append(result) \n</code></pre></p> <pre><code>y_values\n</code></pre> <pre><code>[1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21]\n</code></pre> <p>Convert to numpy array</p> <p>You will slowly get a hang on how when you deal with PyTorch tensors, you just keep on making sure your raw data is in numpy form to make sure everything's good.</p> <pre><code>y_train = np.array(y_values, dtype=np.float32)\ny_train.shape\n</code></pre> <pre><code>(11,)\n</code></pre> <p>Reshape y numpy array to 2-dimension</p> <pre><code># IMPORTANT: 2D required\ny_train = y_train.reshape(-1, 1)\ny_train.shape\n</code></pre> <pre><code>(11, 1)\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#building-model","title":"Building Model","text":"<p>Critical Imports</p> <pre><code>import torch\nimport torch.nn as nn\n</code></pre> <p>Create Model</p> <ol> <li>Linear model<ul> <li>True Equation: \\(y = 2x + 1\\)</li> </ul> </li> <li>Forward<ul> <li>Example<ul> <li>Input \\(x = 1\\)</li> <li>Output \\(\\hat y = ?\\)</li> </ul> </li> </ul> </li> </ol> <pre><code># Create class\nclass LinearRegressionModel(nn.Module):\n    def __init__(self, input_dim, output_dim):\n        super(LinearRegressionModel, self).__init__()\n        self.linear = nn.Linear(input_dim, output_dim)  \n\n    def forward(self, x):\n        out = self.linear(x)\n        return out\n</code></pre> <p>Instantiate Model Class</p> <ul> <li>input: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]</li> <li>desired output: [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21]</li> </ul> <pre><code>input_dim = 1\noutput_dim = 1\n\nmodel = LinearRegressionModel(input_dim, output_dim)\n</code></pre> <p>Instantiate Loss Class</p> <ul> <li>MSE Loss: Mean Squared Error</li> <li>\\(MSE = \\frac{1}{n} \\sum_{i=1}^n(\\hat y_i - y_i)^2\\)<ul> <li>\\(\\hat y\\): prediction</li> <li>\\(y\\): true value</li> </ul> </li> </ul> <pre><code>criterion = nn.MSELoss()\n</code></pre> <p>Instantiate Optimizer Class</p> <ul> <li>Simplified equation<ul> <li>\\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\)<ul> <li>\\(\\theta\\): parameters (our variables)</li> <li>\\(\\eta\\): learning rate (how fast we want to learn)</li> <li>\\(\\nabla_\\theta\\): parameters' gradients</li> </ul> </li> </ul> </li> <li>Even simplier equation<ul> <li><code>parameters = parameters - learning_rate * parameters_gradients</code><ul> <li>parameters: \\(\\alpha\\) and \\(\\beta\\) in \\(y = \\alpha x + \\beta\\)</li> <li>desired parameters: \\(\\alpha = 2\\) and \\(\\beta = 1\\) in \\(y = 2x + 1\\) </li> </ul> </li> </ul> </li> </ul> <pre><code>learning_rate = 0.01\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n</code></pre> <p>Train Model</p> <ul> <li> <p>1 epoch: going through the whole x_train data once</p> <ul> <li>100 epochs: <ul> <li>100x mapping <code>x_train = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]</code></li> </ul> </li> </ul> </li> <li> <p>Process </p> <ol> <li>Convert inputs/labels to tensors with gradients</li> <li>Clear gradient buffets</li> <li>Get output given inputs </li> <li>Get loss</li> <li>Get gradients w.r.t. parameters</li> <li>Update parameters using gradients<ul> <li><code>parameters = parameters - learning_rate * parameters_gradients</code></li> </ul> </li> <li>REPEAT</li> </ol> </li> </ul> <pre><code>epochs = 100\n</code></pre> <pre><code>for epoch in range(epochs):\n    epoch += 1\n    # Convert numpy array to torch Variable\n    inputs = torch.from_numpy(x_train).requires_grad_()\n    labels = torch.from_numpy(y_train)\n\n    # Clear gradients w.r.t. parameters\n    optimizer.zero_grad() \n\n    # Forward to get output\n    outputs = model(inputs)\n\n    # Calculate Loss\n    loss = criterion(outputs, labels)\n\n    # Getting gradients w.r.t. parameters\n    loss.backward()\n\n    # Updating parameters\n    optimizer.step()\n\n    print('epoch {}, loss {}'.format(epoch, loss.item()))\n</code></pre> <pre><code>epoch 1, loss 140.58143615722656\nepoch 2, loss 11.467253684997559\nepoch 3, loss 0.9358152747154236\nepoch 4, loss 0.07679400593042374\nepoch 5, loss 0.0067212567664682865\nepoch 6, loss 0.0010006226366385818\nepoch 7, loss 0.0005289533291943371\nepoch 8, loss 0.0004854927829001099\nepoch 9, loss 0.00047700389404781163\nepoch 10, loss 0.0004714332753792405\nepoch 11, loss 0.00046614606981165707\nepoch 12, loss 0.0004609318566508591\nepoch 13, loss 0.0004557870561257005\nepoch 14, loss 0.00045069155748933554\nepoch 15, loss 0.00044567222357727587\nepoch 16, loss 0.00044068993884138763\nepoch 17, loss 0.00043576463940553367\nepoch 18, loss 0.00043090470717288554\nepoch 19, loss 0.00042609183583408594\nepoch 20, loss 0.0004213254142086953\nepoch 21, loss 0.0004166301223449409\nepoch 22, loss 0.0004119801160413772\nepoch 23, loss 0.00040738462121225893\nepoch 24, loss 0.0004028224211651832\nepoch 25, loss 0.0003983367350883782\nepoch 26, loss 0.0003938761365134269\nepoch 27, loss 0.000389480876037851\nepoch 28, loss 0.00038514015614055097\nepoch 29, loss 0.000380824290914461\nepoch 30, loss 0.00037657516077160835\nepoch 31, loss 0.000372376263840124\nepoch 32, loss 0.0003682126116473228\nepoch 33, loss 0.0003640959912445396\nepoch 34, loss 0.00036003670538775623\nepoch 35, loss 0.00035601368290372193\nepoch 36, loss 0.00035203873994760215\nepoch 37, loss 0.00034810820943675935\nepoch 38, loss 0.000344215368386358\nepoch 39, loss 0.0003403784066904336\nepoch 40, loss 0.00033658024040050805\nepoch 41, loss 0.0003328165039420128\nepoch 42, loss 0.0003291067841928452\nepoch 43, loss 0.0003254293987993151\nepoch 44, loss 0.0003217888588551432\nepoch 45, loss 0.0003182037326041609\nepoch 46, loss 0.0003146533854305744\nepoch 47, loss 0.00031113551813177764\nepoch 48, loss 0.0003076607536058873\nepoch 49, loss 0.00030422292184084654\nepoch 50, loss 0.00030083119054324925\nepoch 51, loss 0.00029746422660537064\nepoch 52, loss 0.0002941471466328949\nepoch 53, loss 0.00029085995629429817\nepoch 54, loss 0.0002876132493838668\nepoch 55, loss 0.00028440452297218144\nepoch 56, loss 0.00028122696676291525\nepoch 57, loss 0.00027808290906250477\nepoch 58, loss 0.00027497278642840683\nepoch 59, loss 0.00027190230321139097\nepoch 60, loss 0.00026887087733484805\nepoch 61, loss 0.0002658693410921842\nepoch 62, loss 0.0002629039518069476\nepoch 63, loss 0.00025996880140155554\nepoch 64, loss 0.0002570618235040456\nepoch 65, loss 0.00025419273879379034\nepoch 66, loss 0.00025135406758636236\nepoch 67, loss 0.0002485490695107728\nepoch 68, loss 0.0002457649679854512\nepoch 69, loss 0.0002430236927466467\nepoch 70, loss 0.00024031475186347961\nepoch 71, loss 0.00023762597993481904\nepoch 72, loss 0.00023497406800743192\nepoch 73, loss 0.0002323519001947716\nepoch 74, loss 0.00022976362379267812\nepoch 75, loss 0.0002271933335578069\nepoch 76, loss 0.00022465786605607718\nepoch 77, loss 0.00022214400814846158\nepoch 78, loss 0.00021966728672850877\nepoch 79, loss 0.0002172116219298914\nepoch 80, loss 0.00021478648704942316\nepoch 81, loss 0.00021239375928416848\nepoch 82, loss 0.0002100227284245193\nepoch 83, loss 0.00020767028036061674\nepoch 84, loss 0.00020534756185952574\nepoch 85, loss 0.00020305956422816962\nepoch 86, loss 0.0002007894654525444\nepoch 87, loss 0.00019854879064951092\nepoch 88, loss 0.00019633043848443776\nepoch 89, loss 0.00019413618429098278\nepoch 90, loss 0.00019197272195015103\nepoch 91, loss 0.0001898303598864004\nepoch 92, loss 0.00018771187751553953\nepoch 93, loss 0.00018561164324637502\nepoch 94, loss 0.00018354636267758906\nepoch 95, loss 0.00018149390234611928\nepoch 96, loss 0.0001794644631445408\nepoch 97, loss 0.00017746571393217891\nepoch 98, loss 0.00017548113828524947\nepoch 99, loss 0.00017352371651213616\nepoch 100, loss 0.00017157981346827\n</code></pre> <p>Looking at predicted values</p> <pre><code># Purely inference\npredicted = model(torch.from_numpy(x_train).requires_grad_()).data.numpy()\npredicted\n</code></pre> <pre><code>array([[ 0.9756333],\n       [ 2.9791424],\n       [ 4.982651 ],\n       [ 6.9861603],\n       [ 8.98967  ],\n       [10.993179 ],\n       [12.996688 ],\n       [15.000196 ],\n       [17.003706 ],\n       [19.007215 ],\n       [21.010725 ]], dtype=float32)\n</code></pre> <p>Looking at training values</p> <p>These are the true values, you can see how it's able to predict similar values.</p> <pre><code># y = 2x + 1 \ny_train\n</code></pre> <pre><code>array([[ 1.],\n       [ 3.],\n       [ 5.],\n       [ 7.],\n       [ 9.],\n       [11.],\n       [13.],\n       [15.],\n       [17.],\n       [19.],\n       [21.]], dtype=float32)\n</code></pre> <p>Plot of predicted and actual values</p> <pre><code># Clear figure\nplt.clf()\n\n# Get predictions\npredicted = model(torch.from_numpy(x_train).requires_grad_()).data.numpy()\n\n# Plot true data\nplt.plot(x_train, y_train, 'go', label='True data', alpha=0.5)\n\n# Plot predictions\nplt.plot(x_train, predicted, '--', label='Predictions', alpha=0.5)\n\n# Legend and plot\nplt.legend(loc='best')\nplt.show()\n</code></pre> <p></p> <p>Save Model</p> <pre><code>save_model = False\nif save_model is True:\n    # Saves only parameters\n    # alpha &amp; beta\n    torch.save(model.state_dict(), 'awesome_model.pkl')\n</code></pre> <p>Load Model</p> <pre><code>load_model = False\nif load_model is True:\n    model.load_state_dict(torch.load('awesome_model.pkl'))\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#building-a-linear-regression-model-with-pytorch-gpu","title":"Building a Linear Regression Model with PyTorch (GPU)","text":"<p>CPU Summary</p> <pre><code>import torch\nimport torch.nn as nn\n\n'''\nSTEP 1: CREATE MODEL CLASS\n'''\nclass LinearRegressionModel(nn.Module):\n    def __init__(self, input_dim, output_dim):\n        super(LinearRegressionModel, self).__init__()\n        self.linear = nn.Linear(input_dim, output_dim)  \n\n    def forward(self, x):\n        out = self.linear(x)\n        return out\n\n'''\nSTEP 2: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 1\noutput_dim = 1\n\nmodel = LinearRegressionModel(input_dim, output_dim)\n\n'''\nSTEP 3: INSTANTIATE LOSS CLASS\n'''\n\ncriterion = nn.MSELoss()\n\n'''\nSTEP 4: INSTANTIATE OPTIMIZER CLASS\n'''\n\nlearning_rate = 0.01\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n\n'''\nSTEP 5: TRAIN THE MODEL\n'''\nepochs = 100\nfor epoch in range(epochs):\n    epoch += 1\n    # Convert numpy array to torch Variable\n    inputs = torch.from_numpy(x_train).requires_grad_()\n    labels = torch.from_numpy(y_train)\n\n    # Clear gradients w.r.t. parameters\n    optimizer.zero_grad() \n\n    # Forward to get output\n    outputs = model(inputs)\n\n    # Calculate Loss\n    loss = criterion(outputs, labels)\n\n    # Getting gradients w.r.t. parameters\n    loss.backward()\n\n    # Updating parameters\n    optimizer.step()\n</code></pre> <p>GPU Summary</p> <ul> <li>Just remember always 2 things must be on GPU<ul> <li><code>model</code></li> <li><code>tensors</code></li> </ul> </li> </ul> <pre><code>import torch\nimport torch.nn as nn\nimport numpy as np\n\n'''\nSTEP 1: CREATE MODEL CLASS\n'''\nclass LinearRegressionModel(nn.Module):\n    def __init__(self, input_dim, output_dim):\n        super(LinearRegressionModel, self).__init__()\n        self.linear = nn.Linear(input_dim, output_dim)  \n\n    def forward(self, x):\n        out = self.linear(x)\n        return out\n\n'''\nSTEP 2: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 1\noutput_dim = 1\n\nmodel = LinearRegressionModel(input_dim, output_dim)\n\n\n#######################\n#  USE GPU FOR MODEL  #\n#######################\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\n'''\nSTEP 3: INSTANTIATE LOSS CLASS\n'''\n\ncriterion = nn.MSELoss()\n\n'''\nSTEP 4: INSTANTIATE OPTIMIZER CLASS\n'''\n\nlearning_rate = 0.01\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n\n'''\nSTEP 5: TRAIN THE MODEL\n'''\nepochs = 100\nfor epoch in range(epochs):\n    epoch += 1\n    # Convert numpy array to torch Variable\n\n    #######################\n    #  USE GPU FOR MODEL  #\n    #######################\n    inputs = torch.from_numpy(x_train).to(device)\n    labels = torch.from_numpy(y_train).to(device)\n\n    # Clear gradients w.r.t. parameters\n    optimizer.zero_grad() \n\n    # Forward to get output\n    outputs = model(inputs)\n\n    # Calculate Loss\n    loss = criterion(outputs, labels)\n\n    # Getting gradients w.r.t. parameters\n    loss.backward()\n\n    # Updating parameters\n    optimizer.step()\n\n    # Logging\n    print('epoch {}, loss {}'.format(epoch, loss.item()))\n</code></pre> <pre><code>epoch 1, loss 336.0314025878906\nepoch 2, loss 27.67657470703125\nepoch 3, loss 2.5220539569854736\nepoch 4, loss 0.46732547879219055\nepoch 5, loss 0.2968060076236725\nepoch 6, loss 0.2800087630748749\nepoch 7, loss 0.27578213810920715\nepoch 8, loss 0.2726128399372101\nepoch 9, loss 0.269561231136322\nepoch 10, loss 0.2665504515171051\nepoch 11, loss 0.2635740041732788\nepoch 12, loss 0.26063060760498047\nepoch 13, loss 0.2577202618122101\nepoch 14, loss 0.2548423111438751\nepoch 15, loss 0.25199657678604126\nepoch 16, loss 0.24918246269226074\nepoch 17, loss 0.24639996886253357\nepoch 18, loss 0.24364829063415527\nepoch 19, loss 0.24092751741409302\nepoch 20, loss 0.2382371574640274\nepoch 21, loss 0.23557686805725098\nepoch 22, loss 0.2329462170600891\nepoch 23, loss 0.2303449958562851\nepoch 24, loss 0.22777271270751953\nepoch 25, loss 0.2252292037010193\nepoch 26, loss 0.22271405160427094\nepoch 27, loss 0.22022713720798492\nepoch 28, loss 0.21776780486106873\nepoch 29, loss 0.21533599495887756\nepoch 30, loss 0.21293145418167114\nepoch 31, loss 0.21055366098880768\nepoch 32, loss 0.20820240676403046\nepoch 33, loss 0.2058774083852768\nepoch 34, loss 0.20357847213745117\nepoch 35, loss 0.20130516588687897\nepoch 36, loss 0.1990572065114975\nepoch 37, loss 0.19683438539505005\nepoch 38, loss 0.19463638961315155\nepoch 39, loss 0.19246290624141693\nepoch 40, loss 0.1903136670589447\nepoch 41, loss 0.1881885528564453\nepoch 42, loss 0.18608702719211578\nepoch 43, loss 0.18400898575782776\nepoch 44, loss 0.18195408582687378\nepoch 45, loss 0.17992223799228668\nepoch 46, loss 0.17791320383548737\nepoch 47, loss 0.17592646181583405\nepoch 48, loss 0.17396186292171478\nepoch 49, loss 0.17201924324035645\nepoch 50, loss 0.17009828984737396\nepoch 51, loss 0.16819894313812256\nepoch 52, loss 0.16632060706615448\nepoch 53, loss 0.16446338593959808\nepoch 54, loss 0.16262666881084442\nepoch 55, loss 0.16081078350543976\nepoch 56, loss 0.15901507437229156\nepoch 57, loss 0.15723931789398193\nepoch 58, loss 0.15548335015773773\nepoch 59, loss 0.15374726057052612\nepoch 60, loss 0.1520303338766098\nepoch 61, loss 0.15033268928527832\nepoch 62, loss 0.14865389466285706\nepoch 63, loss 0.14699392020702362\nepoch 64, loss 0.14535246789455414\nepoch 65, loss 0.14372935891151428\nepoch 66, loss 0.14212435483932495\nepoch 67, loss 0.14053721725940704\nepoch 68, loss 0.13896773755550385\nepoch 69, loss 0.1374160647392273\nepoch 70, loss 0.1358814686536789\nepoch 71, loss 0.13436420261859894\nepoch 72, loss 0.13286370038986206\nepoch 73, loss 0.1313801407814026\nepoch 74, loss 0.12991292774677277\nepoch 75, loss 0.12846232950687408\nepoch 76, loss 0.1270277351140976\nepoch 77, loss 0.12560924887657166\nepoch 78, loss 0.12420656532049179\nepoch 79, loss 0.12281957268714905\nepoch 80, loss 0.1214480847120285\nepoch 81, loss 0.12009195983409882\nepoch 82, loss 0.1187509223818779\nepoch 83, loss 0.11742479354143143\nepoch 84, loss 0.11611353605985641\nepoch 85, loss 0.11481687426567078\nepoch 86, loss 0.11353478580713272\nepoch 87, loss 0.11226697266101837\nepoch 88, loss 0.11101329326629639\nepoch 89, loss 0.10977360606193542\nepoch 90, loss 0.10854770988225937\nepoch 91, loss 0.10733554512262344\nepoch 92, loss 0.10613703727722168\nepoch 93, loss 0.10495180636644363\nepoch 94, loss 0.10377981513738632\nepoch 95, loss 0.10262089222669601\nepoch 96, loss 0.10147502273321152\nepoch 97, loss 0.1003417894244194\nepoch 98, loss 0.09922132641077042\nepoch 99, loss 0.0981132984161377\nepoch 100, loss 0.09701769798994064\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#summary","title":"Summary","text":"<p>We've learnt to...</p> <p>Success</p> <ul> <li> Simple linear regression basics<ul> <li> \\(y = Ax + B\\)</li> <li> \\(y = 2x + 1\\)</li> </ul> </li> <li> Example of simple linear regression</li> <li> Aim of linear regression<ul> <li> Minimizing distance between the points and the line<ul> <li> Calculate \"distance\" through <code>MSE</code></li> <li> Calculate <code>gradients</code></li> <li> Update parameters with <code>parameters = parameters - learning_rate * gradients</code></li> <li> Slowly update parameters \\(A\\) and \\(B\\) model the linear relationship between \\(y\\) and \\(x\\) of the form \\(y = 2x + 1\\)</li> </ul> </li> </ul> </li> <li> Built a linear regression model in CPU and GPU<ul> <li> Step 1: Create Model Class</li> <li> Step 2: Instantiate Model Class</li> <li> Step 3: Instantiate Loss Class</li> <li> Step 4: Instantiate Optimizer Class</li> <li> Step 5: Train Model</li> </ul> </li> <li> Important things to be on GPU<ul> <li> <code>model</code></li> <li> <code>tensors with gradients</code></li> </ul> </li> <li> How to bring to GPU?<ul> <li><code>model_name.to(device)</code></li> <li><code>variable_name.to(device)</code></li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_linear_regression/#citation","title":"Citation","text":"<p>If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.</p> <p></p>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/","title":"Logistic Regression with PyTorch","text":"<p>Run Jupyter Notebook</p> <p>You can run the code for this section in this jupyter notebook link.</p>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#about-logistic-regression","title":"About Logistic Regression","text":""},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#logistic-regression-basics","title":"Logistic Regression Basics","text":""},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#classification-algorithm","title":"Classification algorithm","text":"<ul> <li>Example: Spam vs No Spam<ul> <li>Input: Bunch of words</li> <li>Output: Probability spam or not</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#basic-comparison","title":"Basic Comparison","text":"<ul> <li>Linear regression<ul> <li>Output: numeric value given inputs</li> </ul> </li> <li>Logistic regression:<ul> <li>Output: probability [0, 1] given input belonging to a class</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#inputoutput-comparison","title":"Input/Output Comparison","text":"<ul> <li>Linear regression: Multiplication<ul> <li>Input: [1]<ul> <li>Output: 2</li> </ul> </li> <li>Input: [2]<ul> <li>Output: 4</li> </ul> </li> <li>Trying to model the relationship <code>y = 2x</code></li> </ul> </li> <li>Logistic regression: Spam<ul> <li>Input: \"Sign up to get 1 million dollars by tonight\"<ul> <li>Output: p = 0.8</li> </ul> </li> <li>Input: \"This is a receipt for your recent purchase with Amazon\"<ul> <li>Output: p = 0.3</li> </ul> </li> <li>p: probability it is spam</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#problems-of-linear-regression","title":"Problems of Linear Regression","text":"<ul> <li>Example<ul> <li>Fever</li> <li>Input: temperature</li> <li>Output: fever or no fever</li> </ul> </li> <li>Remember<ul> <li>Linear regression: minimize error between points and line</li> </ul> </li> </ul> <p>Linear Regression Problem 1: Fever value can go negative (below 0) and positive (above 1)</p> <p>If you simply tried to do a simple linear regression on this fever problem, you would realize an apparent error. Fever can go beyond 1 and below 0 which does not make sense in this context. <pre><code>import numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nx = [1, 5, 10, 10, 25, 50, 70, 75, 100,]\ny = [0, 0, 0, 0, 0, 1, 1, 1, 1]\n\ncolors = np.random.rand(len(x))\nplt.plot(np.unique(x), np.poly1d(np.polyfit(x, y, 1))(np.unique(x)))\nplt.ylabel(\"Fever\")\nplt.xlabel(\"Temperature\")\n\nplt.scatter(x, y, c=colors, alpha=0.5)\nplt.show()\n</code></pre></p> <p></p> <p>Linear Regression Problem 2: Fever points are not predicted with the presence of outliers</p> <p>Previously at least some points could be properly predicted. However, with the presence of outliers, everything goes wonky for simple linear regression, having no predictive capacity at all. <pre><code>import numpy as np\nimport matplotlib.pyplot as plt\n\nx = [1, 5, 10, 10, 25, 50, 70, 75, 300]\ny = [0, 0, 0, 0, 0, 1, 1, 1, 1]\n\ncolors = np.random.rand(len(x))\nplt.plot(np.unique(x), np.poly1d(np.polyfit(x, y, 1))(np.unique(x)))\nplt.ylabel(\"Fever\")\nplt.xlabel(\"Temperature\")\n\nplt.scatter(x, y, c=colors, alpha=0.5)\nplt.show()\n</code></pre></p> <p></p>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#logistic-regression-in-depth","title":"Logistic Regression In-Depth","text":""},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#predicting-probability","title":"Predicting Probability","text":"<ul> <li>Linear regression doesn't work</li> <li>Instead of predicting direct values: predict probability</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#logistic-function-g","title":"Logistic Function g()","text":"<ul> <li>\"Two-class logistic regression\"</li> <li>\\(\\boldsymbol{y} = A\\boldsymbol{x} + \\boldsymbol{b}\\)<ul> <li>Where \\(\\boldsymbol{y}\\) is a vector comprising the 2-class prediction \\(y_0\\) and \\(y_1\\)</li> <li>Where the labels are \\(y_0 = 0\\)  and \\(y_1 = 1\\)</li> <li>Also, it's bolded because it's a vector, not a matrix.</li> </ul> </li> <li>\\(g(y_1) = \\frac {1} {1 + e^{-y_1}}\\)<ul> <li>\\(g(y_1)\\) = Estimated probability that \\(y = 1\\)</li> </ul> </li> <li>\\(g(y_0) = 1 - g(y_1)\\)<ul> <li>\\(g(y_0)\\) = Estimated probability that \\(y = 0\\)</li> </ul> </li> <li>For our illustration above, we have 4 classes, so we have to use softmax function explained below</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#softmax-function-g","title":"Softmax Function g()","text":"<ul> <li>\"Multi-class logistic regression\"<ul> <li>Generalization of logistic function, where you can derive back to the logistic function if you've a 2 class classification problem</li> <li>Here, we will use a 4 class example (K = 4) as shown above to be very clear in how it relates back to that simple examaple.</li> </ul> </li> <li>\\(\\boldsymbol{y} = A\\boldsymbol{x} + \\boldsymbol{b}\\)<ul> <li>Where \\(\\boldsymbol{y}\\) is a vector comprising the 4-class prediction \\(y_0, y_1, y_2, y_3\\)</li> <li>Where the 4 labels (K = 4) are \\(y_0 = 0, y_1 = 1, y_2 = 2, y_3 = 3\\)</li> </ul> </li> <li>\\(g(y_i) = \\frac {e^{y_i} } {\\sum^K_i e^{y_i}}\\) where K = 4 because we have 4 classes<ul> <li>To put numbers to this equation in relation to the illustration above where we've \\(y_0 = 1.3, y_1 = 1.2, y = 4.5, y = 4.8\\)<ul> <li>\\(g(y_0) = \\frac {e^{1.3}}{e^{1.3} + e^{1.2} + e^{4.5} + e^{4.8}} = 0.017\\)</li> <li>\\(g(y_1) = \\frac {e^{1.2}}{e^{1.3} + e^{1.2} + e^{4.5} + e^{4.8}} = 0.015\\)</li> <li>\\(g(y_2) = \\frac {e^{4.5}}{e^{1.3} + e^{1.2} + e^{4.5} + e^{4.8}} = 0.412\\)</li> <li>\\(g(y_3) = \\frac {e^{4.8}}{e^{1.3} + e^{1.2} + e^{4.5} + e^{4.8}} = 0.556\\)</li> <li>\\(g(y_0) + g(y_1) + g(y_2) + g(y_3) = 1.0\\)</li> <li>All softmax outputs have to sum to one as they represent a probability distribution over K classes. </li> </ul> </li> </ul> </li> <li>Take note how these numbers are not exactly as in the illustration in the softmax box but the concept is important (intentionally made so).<ul> <li>\\(y_0\\) and \\(y_1\\) are approximately similar in values and they return similar probabilities.</li> <li>Similarly, \\(y_2\\) and \\(y_3\\) are approximately similar in values and they return similar probabilities.</li> </ul> </li> </ul> <p>Softmax versus Soft(arg)max</p> <p>Do you know many researchers and anyone in deep learning in general use the term softmax when it should be soft(arg)max.</p> <p>This is because soft(arg)max returns the probability distribution over K classes, a vector. </p> <p>However, softmax only returns the max! This means you will be getting a scalar value versus a probability distribution.</p> <p>According to my friend, Alfredo Canziani (postdoc in NYU under Yann Lecun), it was actually a mistake made in the original paper previously but it was too late because the term softmax was adopted. Full credits to him for this tip.</p>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#cross-entropy-function-d-for-2-class","title":"Cross Entropy Function D() for 2 Class","text":"<ul> <li>Take note that here, \\(S\\) is our softmax outputs and \\(L\\) are our labels</li> <li>\\(D(S, L) = -(L log S + (1-L)log(1-S))\\)<ul> <li>If L = 0 (label)<ul> <li>\\(D(S, 0) = - log(1-S)\\)<ul> <li>\\(- log(1-S)\\): less positive if \\(S \\longrightarrow 0\\)</li> <li>\\(- log(1-S)\\): more positive if \\(S \\longrightarrow 1\\) (BIGGER LOSS)</li> </ul> </li> </ul> </li> <li>If L = 1 (label)<ul> <li>\\(D(S, 1) = - log S\\)<ul> <li>\\(-log(S)\\): less positive if \\(S \\longrightarrow 1\\)</li> <li>\\(-log(S)\\): more positive if \\(S \\longrightarrow 0\\) (BIGGER LOSS)</li> </ul> </li> </ul> </li> </ul> </li> </ul> <p>Numerical example of bigger or small loss</p> <p>You get a small error of 1e-5 if your label = 0 and your S is closer to 0 (very correct prediction). <pre><code>import math\nprint(-math.log(1 - 0.00001))\n</code></pre></p> <p>You get a large error of 11.51 if your label is 0 and S is near to 1 (very wrong prediction). <pre><code>print(-math.log(1 - 0.99999)) \n</code></pre></p> <p>You get a small error of -1e-5 if your label is 1 and S is near 1 (very correct prediction). <pre><code>print(-math.log(0.99999))\n</code></pre></p> <p>You get a big error of -11.51 if your label is 1 and S is near 0 (very wrong prediction). <pre><code>print(-math.log(0.00001))\n</code></pre></p> <pre><code>1.0000050000287824e-05\n11.51292546497478\n1.0000050000287824e-05\n11.512925464970229\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#cross-entropy-function-d-for-more-than-2-class","title":"Cross Entropy Function D() for More Than 2 Class","text":"<ul> <li>For the case where we have more than 2 class, we need a more generalized function</li> <li>\\(D(S, L) = - \\sum^K_1 L_i log(S_i)\\)<ul> <li>\\(K\\): number of classes</li> <li>\\(L_i\\): label of i-th class, 1 if that's the class else 0</li> <li>\\(S_i\\): output of softmax for i-th class</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#cross-entropy-loss-over-n-samples","title":"Cross Entropy Loss over N samples","text":"<ul> <li>Goal: Minimizing Cross Entropy Loss, L</li> <li>\\(Loss = \\frac {1}{N} \\sum_j^N D_j\\)<ul> <li>\\(D_j\\): j-th sample of cross entropy function \\(D(S, L)\\)</li> <li>\\(N\\): number of samples</li> <li>\\(Loss\\): average cross entropy loss over N samples</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#building-a-logistic-regression-model-with-pytorch","title":"Building a Logistic Regression Model with PyTorch","text":""},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#steps","title":"Steps","text":"<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#step-1a-loading-mnist-train-dataset","title":"Step 1a: Loading MNIST Train Dataset","text":"<p>Images from 1 to 9</p> <p>Inspect length of training dataset</p> <p>You can easily load MNIST dataset with PyTorch. Here we inspect the training set, where our algorithms will learn from, and you will discover it is made up of 60,000 images. <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n</code></pre></p> <pre><code>train_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n</code></pre> <pre><code>len(train_dataset)\n</code></pre> <pre><code>60000\n</code></pre> <p>Inspecting a single image</p> <p>So this is how a single image is represented in numbers. It's actually a 28 pixel x 28 pixel image which is why you would end up with this 28x28 matrix of numbers. <pre><code>train_dataset[0]\n</code></pre></p> <pre><code>(tensor([[[ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0118,  0.0706,\n            0.0706,  0.0706,  0.4941,  0.5333,  0.6863,  0.1020,  0.6510,\n            1.0000,  0.9686,  0.4980,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.1176,  0.1412,  0.3686,  0.6039,  0.6667,  0.9922,\n            0.9922,  0.9922,  0.9922,  0.9922,  0.8824,  0.6745,  0.9922,\n            0.9490,  0.7647,  0.2510,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.1922,  0.9333,  0.9922,  0.9922,  0.9922,  0.9922,  0.9922,\n            0.9922,  0.9922,  0.9922,  0.9843,  0.3647,  0.3216,  0.3216,\n            0.2196,  0.1529,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0706,  0.8588,  0.9922,  0.9922,  0.9922,  0.9922,  0.9922,\n            0.7765,  0.7137,  0.9686,  0.9451,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.3137,  0.6118,  0.4196,  0.9922,  0.9922,  0.8039,\n            0.0431,  0.0000,  0.1686,  0.6039,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0549,  0.0039,  0.6039,  0.9922,  0.3529,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.5451,  0.9922,  0.7451,\n            0.0078,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0431,  0.7451,  0.9922,\n            0.2745,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.1373,  0.9451,\n            0.8824,  0.6275,  0.4235,  0.0039,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.3176,\n            0.9412,  0.9922,  0.9922,  0.4667,  0.0980,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.1765,  0.7294,  0.9922,  0.9922,  0.5882,  0.1059,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0627,  0.3647,  0.9882,  0.9922,  0.7333,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.9765,  0.9922,  0.9765,  0.2510,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.1804,  0.5098,  0.7176,  0.9922,  0.9922,  0.8118,  0.0078,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.1529,  0.5804,\n            0.8980,  0.9922,  0.9922,  0.9922,  0.9804,  0.7137,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0941,  0.4471,  0.8667,  0.9922,\n            0.9922,  0.9922,  0.9922,  0.7882,  0.3059,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0902,  0.2588,  0.8353,  0.9922,  0.9922,  0.9922,\n            0.9922,  0.7765,  0.3176,  0.0078,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0706,\n            0.6706,  0.8588,  0.9922,  0.9922,  0.9922,  0.9922,  0.7647,\n            0.3137,  0.0353,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.2157,  0.6745,  0.8863,\n            0.9922,  0.9922,  0.9922,  0.9922,  0.9569,  0.5216,  0.0431,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.5333,  0.9922,  0.9922,\n            0.9922,  0.8314,  0.5294,  0.5176,  0.0627,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,\n            0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000]]]),\n tensor(5))\n</code></pre> <p>Inspecting a single data point in the training dataset</p> <p>When you load MNIST dataset, each data point is actually a tuple containing the image matrix and the label.</p> <pre><code>type(train_dataset[0])\n</code></pre> <pre><code>tuple\n</code></pre> <p>Inspecting training dataset first element of tuple</p> <p>This means to access the image, you need to access the first element in the tuple.</p> <pre><code># Input Matrix\ntrain_dataset[0][0].size()\n</code></pre> <pre><code># A 28x28 sized image of a digit\ntorch.Size([1, 28, 28])\n</code></pre> <p>Inspecting training dataset second element of tuple</p> <p>The second element actually represents the image's label. Meaning if the second element says 5, it means the 28x28 matrix of numbers represent a digit 5.</p> <pre><code># Label\ntrain_dataset[0][1]\n</code></pre> <pre><code>tensor(5)\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#displaying-mnist","title":"Displaying MNIST","text":"<p>Verifying shape of MNIST image</p> <p>As mentioned, a single MNIST image is of the shape 28 pixel x 28 pixel.</p> <pre><code>import matplotlib.pyplot as plt\n%matplotlib inline  \nimport numpy as np\n</code></pre> <pre><code>train_dataset[0][0].numpy().shape\n</code></pre> <pre><code>(1, 28, 28)\n</code></pre> <p>Plot image of MNIST image</p> <pre><code>show_img = train_dataset[0][0].numpy().reshape(28, 28)\n</code></pre> <pre><code>plt.imshow(show_img, cmap='gray')\n</code></pre> <p></p> <p>Second element of tuple shows label</p> <p>As you would expect, the label is 5. <pre><code># Label\ntrain_dataset[0][1]\n</code></pre></p> <pre><code>tensor(5)\n</code></pre> <p>Plot second image of MNIST image</p> <pre><code>show_img = train_dataset[1][0].numpy().reshape(28, 28)\n</code></pre> <pre><code>plt.imshow(show_img, cmap='gray')\n</code></pre> <p></p> <p>Second element of tuple shows label</p> <p>We should see 0 here as the label. <pre><code># Label\ntrain_dataset[1][1]\n</code></pre></p> <pre><code>tensor(0)\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#step-1b-loading-mnist-test-dataset","title":"Step 1b: Loading MNIST Test Dataset","text":"<ul> <li>Show our algorithm works beyond the data we have trained on.</li> <li>Out-of-sample</li> </ul> <p>Load test dataset</p> <p>Compared to the 60k images in the training set, the testing set where the model will not be trained on has 10k images to check for its out-of-sample performance. <pre><code>test_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n</code></pre></p> <pre><code>len(test_dataset)\n</code></pre> <pre><code>10000\n</code></pre> <p>Test dataset elements</p> <p>Exactly like the training set, the testing set has 10k tuples containing the 28x28 matrices and their respective labels. <pre><code>type(test_dataset[0])\n</code></pre></p> <pre><code>tuple\n</code></pre> <p>Test dataset first element in tuple</p> <p>This contains the image matrix, similar to the training set. <pre><code># Image matrix\ntest_dataset[0][0].size()\n</code></pre></p> <pre><code>torch.Size([1, 28, 28])\n</code></pre> <p>Plot image sample from test dataset</p> <pre><code>show_img = test_dataset[0][0].numpy().reshape(28, 28)\nplt.imshow(show_img, cmap='gray')\n</code></pre> <p></p> <p>Test dataset second element in tuple</p> <pre><code># Label\ntest_dataset[0][1]\n</code></pre> <pre><code>tensor(7)\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#step-2-make-dataset-iterable","title":"Step 2: Make Dataset Iterable","text":"<ul> <li>Aim: make the dataset iterable</li> <li>totaldata: 60000</li> <li>minibatch: 100<ul> <li>Number of examples in 1 iteration</li> </ul> </li> <li>iterations: 3000<ul> <li>1 iteration: one mini-batch forward &amp; backward pass</li> </ul> </li> <li>epochs<ul> <li>1 epoch: running through the whole dataset once</li> <li>\\(epochs = iterations \\div \\frac{totaldata}{minibatch} = 3000 \\div \\frac{60000}{100} = 5\\)</li> </ul> </li> </ul> <p>Recap training dataset</p> <p>Remember training dataset has 60k images and testing dataset has 10k images. <pre><code>len(train_dataset)\n</code></pre></p> <pre><code>60000\n</code></pre> <p>Defining epochs</p> <p>When the model goes through the whole 60k images once, learning how to classify 0-9, it's consider 1 epoch. </p> <p>However, there's a concept of batch size where it means the model would look at 100 images before updating the model's weights, thereby learning. When the model updates its weights (parameters) after looking at all the images, this is considered 1 iteration.</p> <pre><code>batch_size = 100\n</code></pre> <p>We arbitrarily set 3000 iterations here which means the model would update 3000 times. <pre><code>n_iters = 3000\n</code></pre></p> <p>One epoch consists of 60,000 / 100 = 600 iterations. Because we would like to go through 3000 iterations, this implies we would have 3000 / 600 = 5 epochs as each epoch has 600 iterations. </p> <pre><code>num_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\nnum_epochs\n</code></pre> <pre><code>5\n</code></pre> <p>Create Iterable Object: Training Dataset</p> <pre><code>train_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n</code></pre> <p>Check Iterability</p> <pre><code>import collections\nisinstance(train_loader, collections.Iterable)\n</code></pre> <pre><code>True\n</code></pre> <p>Create Iterable Object: Testing Dataset</p> <pre><code># Iterable object\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n</code></pre> <p>Check iterability of testing dataset</p> <pre><code>isinstance(test_loader, collections.Iterable)\n</code></pre> <pre><code>True\n</code></pre> <p>Iterate through dataset</p> <p>This is just a simplified example of what we're doing above where we're creating an iterable object <code>lst</code> to loop through so we can access all the images <code>img_1</code> and <code>img_2</code>.</p> <p>Above, the equivalent of <code>lst</code> is <code>train_loader</code> and <code>test_loader</code>.</p> <pre><code>img_1 = np.ones((28, 28))\nimg_2 = np.ones((28, 28))\nlst = [img_1, img_2]\n</code></pre> <pre><code># Need to iterate\n# Think of numbers as the images\nfor i in lst:\n    print(i.shape)\n</code></pre> <pre><code>(28, 28)\n(28, 28)\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#step-3-building-model","title":"Step 3: Building Model","text":"<p>Create model class</p> <pre><code># Same as linear regression! \nclass LogisticRegressionModel(nn.Module):\n    def __init__(self, input_dim, output_dim):\n        super(LogisticRegressionModel, self).__init__()\n        self.linear = nn.Linear(input_dim, output_dim)\n\n    def forward(self, x):\n        out = self.linear(x)\n        return out\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#step-4-instantiate-model-class","title":"Step 4: Instantiate Model Class","text":"<ul> <li>Input dimension: <ul> <li>Size of image</li> <li>\\(28 \\times 28 = 784\\)</li> </ul> </li> <li>Output dimension: 10<ul> <li>0, 1, 2, 3, 4, 5, 6, 7, 8, 9</li> </ul> </li> </ul> <p>Check size of dataset</p> <p>This should be 28x28. <pre><code># Size of images\ntrain_dataset[0][0].size()\n</code></pre></p> <pre><code>torch.Size([1, 28, 28])\n</code></pre> <p>Instantiate model class based on input and out dimensions</p> <p>As we're trying to classify digits 0-9 a total of 10 classes, our output dimension is 10. </p> <p>And we're feeding the model with 28x28 images, hence our input dimension is 28x28. <pre><code>input_dim = 28*28\noutput_dim = 10\n\nmodel = LogisticRegressionModel(input_dim, output_dim)\n</code></pre></p>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#step-5-instantiate-loss-class","title":"Step 5: Instantiate Loss Class","text":"<ul> <li>Logistic Regression: Cross Entropy Loss<ul> <li>Linear Regression: MSE</li> </ul> </li> </ul> <p>Create Cross Entry Loss Class</p> <p>Unlike linear regression, we do not use MSE here, we need Cross Entry Loss to calculate our loss before we backpropagate and update our parameters.</p> <pre><code>criterion = nn.CrossEntropyLoss()  \n</code></pre> <p>What happens in nn.CrossEntropyLoss()?</p> <p>It does 2 things at the same time. </p> <p> 1. Computes softmax (logistic/softmax function)  2. Computes cross entropy</p> <p></p>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#step-6-instantiate-optimizer-class","title":"Step 6: Instantiate Optimizer Class","text":"<ul> <li>Simplified equation<ul> <li>\\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\)<ul> <li>\\(\\theta\\): parameters (our variables)</li> <li>\\(\\eta\\): learning rate (how fast we want to learn)</li> <li>\\(\\nabla_\\theta\\): parameters' gradients</li> </ul> </li> </ul> </li> <li>Even simplier equation<ul> <li><code>parameters = parameters - learning_rate * parameters_gradients</code></li> <li>At every iteration, we update our model's parameters</li> </ul> </li> </ul> <p>Create optimizer</p> <p>Similar to what we've covered above, this calculates the parameters' gradients and update them subsequently. <pre><code>learning_rate = 0.001\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)  \n</code></pre></p> <p>Parameters In-Depth</p> <p>You'll realize we have 2 sets of parameters, 10x784 which is A and 10x1 which is b in the \\(y = AX + b\\) equation where X is our input of size 784.</p> <p>We'll go into details subsequently how these parameters interact with our input to produce our 10x1 output. </p> <pre><code># Type of parameter object\nprint(model.parameters())\n\n# Length of parameters\nprint(len(list(model.parameters())))\n\n# FC 1 Parameters \nprint(list(model.parameters())[0].size())\n\n# FC 1 Bias Parameters\nprint(list(model.parameters())[1].size())\n</code></pre> <pre><code>&lt;generator object Module.parameters at 0x7ff7c884f830&gt;\n2\ntorch.Size([10, 784])\ntorch.Size([10])\n</code></pre> <p>Quick Matrix Product Review</p> <ul> <li>Example 1: matrix product<ul> <li>\\(A: (100, 10)\\)</li> <li>\\(B: (10, 1)\\)</li> <li>\\(A \\cdot B = (100, 10) \\cdot (10, 1) = (100, 1)\\)</li> </ul> </li> <li>Example 2: matrix product<ul> <li>\\(A: (50, 5)\\)</li> <li>\\(B: (5, 2)\\)</li> <li>\\(A \\cdot B = (50, 5) \\cdot (5, 2) = (50, 2)\\)</li> </ul> </li> <li>Example 3: element-wise addition<ul> <li>\\(A: (10, 1)\\)</li> <li>\\(B: (10, 1)\\)</li> <li>\\(A + B = (10, 1)\\)</li> </ul> </li> </ul> <p></p>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#step-7-train-model","title":"Step 7: Train Model","text":"<p>7 step process for training models</p> <ul> <li>Process <ol> <li>Convert inputs/labels to tensors with gradients</li> <li>Clear gradient buffets</li> <li>Get output given inputs</li> <li>Get loss</li> <li>Get gradients w.r.t. parameters</li> <li>Update parameters using gradients<ul> <li><code>parameters = parameters - learning_rate * parameters_gradients</code></li> </ul> </li> <li>REPEAT</li> </ol> </li> </ul> <pre><code>iter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as Variable\n        images = images.view(-1, 28*28).requires_grad_()\n        labels = labels\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, 28*28).requires_grad_()\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 1.8513233661651611. Accuracy: 70\nIteration: 1000. Loss: 1.5732524394989014. Accuracy: 77\nIteration: 1500. Loss: 1.3840199708938599. Accuracy: 79\nIteration: 2000. Loss: 1.1711134910583496. Accuracy: 81\nIteration: 2500. Loss: 1.1094708442687988. Accuracy: 82\nIteration: 3000. Loss: 1.002761721611023. Accuracy: 82\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#break-down-accuracy-calculation","title":"Break Down Accuracy Calculation","text":"<p>Printing outputs of our model</p> <p>As we've trained our model, we can extract the accuracy calculation portion to understand what's happening without re-training the model.</p> <p>This would print out the output of the model's predictions on your notebook.</p> <pre><code>iter_test = 0\nfor images, labels in test_loader:\n    iter_test += 1\n    images = images.view(-1, 28*28).requires_grad_()\n    outputs = model(images)\n    if iter_test == 1:\n        print('OUTPUTS')\n        print(outputs)\n    _, predicted = torch.max(outputs.data, 1)\n</code></pre> <pre><code>OUTPUTS\ntensor([[-0.4181, -1.0784, -0.4840, -0.0985, -0.2394, -0.1801, -1.1639,\n          2.9352, -0.1552,  0.8852],\n        [ 0.5117, -0.1099,  1.5295,  0.8863, -1.8813,  0.5967,  1.3632,\n         -1.8977,  0.4183, -1.4990],\n        [-1.0126,  2.4112,  0.2373,  0.0857, -0.7007, -0.2015, -0.3428,\n         -0.2548,  0.1659, -0.4703],\n        [ 2.8072, -2.2973, -0.0984, -0.4313, -0.9619,  0.8670,  1.2201,\n          0.3752, -0.2873, -0.3272],\n        [-0.0343, -2.0043,  0.5081, -0.6452,  1.8647, -0.6924,  0.1435,\n          0.4330,  0.2958,  1.0339],\n        [-1.5392,  2.9070,  0.2297,  0.3139, -0.6863, -0.2734, -0.8377,\n         -0.1238,  0.3285, -0.3004],\n        [-1.2037, -1.3739, -0.5947,  0.3530,  1.4205,  0.0593, -0.7307,\n          0.6642,  0.3937,  0.8004],\n        [-1.4439, -0.3284, -0.7652, -0.0952,  0.9323,  0.3006,  0.0238,\n         -0.0810,  0.0612,  1.3295],\n        [ 0.5409, -0.5266,  0.9914, -1.2369,  0.6583,  0.0992,  0.8525,\n         -1.0562,  0.2013,  0.0462],\n        [-0.6548, -0.7253, -0.9825, -1.1663,  0.9076, -0.0694, -0.3708,\n          1.8270,  0.2457,  1.5921],\n        [ 3.2147, -1.7689,  0.8531,  1.2320, -0.8126,  1.1251, -0.2776,\n         -1.4244,  0.5930, -1.6183],\n        [ 0.7470, -0.5545,  1.0251,  0.0529,  0.4384, -0.5934,  0.7666,\n         -1.0084,  0.5313, -0.3465],\n        [-0.7916, -1.7064, -0.7805, -1.1588,  1.3284, -0.1708, -0.2092,\n          0.9495,  0.1033,  2.0208],\n        [ 3.0602, -2.3578, -0.2576, -0.2198, -0.2372,  0.9765, -0.1514,\n         -0.5380,  0.7970,  0.1374],\n        [-1.2613,  2.8594, -0.0874,  0.1974, -1.2018, -0.0064, -0.0923,\n         -0.2142,  0.2575, -0.3218],\n        [ 0.4348, -0.7216,  0.0021,  1.2864, -0.5062,  0.7761, -0.3236,\n         -0.5667,  0.5431, -0.7781],\n        [-0.2157, -2.0200,  0.1829, -0.6882,  1.3815, -0.7609, -0.0902,\n          0.8647,  0.3679,  1.8843],\n        [ 0.0950, -1.5009, -0.6347,  0.3662, -0.4679, -0.0359, -0.7671,\n          2.7155, -0.3991,  0.5737],\n        [-0.7005, -0.5366, -0.0434,  1.1289, -0.5873,  0.2555,  0.8187,\n         -0.6557,  0.1241, -0.4297],\n        [-1.0635, -1.5991, -0.4677, -0.1231,  2.0445,  0.1128, -0.1825,\n          0.1075,  0.0348,  1.4317],\n        [-1.0319, -0.1595, -1.3415,  0.1095,  0.5339,  0.1973, -1.3272,\n          1.5765,  0.4784,  1.4176],\n        [-0.4928, -1.5653, -0.0672,  0.3325,  0.5359,  0.5368,  2.1542,\n         -1.4276,  0.3605,  0.0587],\n        [-0.4761,  0.2958,  0.6597, -0.2658,  1.1279, -1.0676,  1.2506,\n         -0.2059, -0.1489,  0.1051],\n        [-0.0764, -0.9274, -0.6838,  0.3464, -0.2656,  1.4099,  0.4486,\n         -0.9527,  0.5682,  0.0156],\n        [-0.6900, -0.9611,  0.1395, -0.0079,  1.5424, -0.3208, -0.2682,\n          0.3586, -0.2771,  1.0389],\n        [ 4.3606, -2.8621,  0.6310, -0.9657, -0.2486,  1.2009,  1.1873,\n         -0.8255, -0.2103, -1.2172],\n        [-0.1000, -1.4268, -0.4627, -0.1041,  0.2959, -0.1392, -0.6855,\n          1.8622, -0.2580,  1.1347],\n        [-0.3625, -2.1323, -0.2224, -0.8754,  2.4684,  0.0295,  0.1161,\n         -0.2660,  0.3037,  1.4570],\n        [ 2.8688, -2.4517,  0.1782,  1.1149, -1.0898,  1.1062, -0.0681,\n         -0.5697,  0.8888, -0.6965],\n        [-1.0429,  1.4446, -0.3349,  0.1254, -0.5017,  0.2286,  0.2328,\n         -0.3290,  0.3949, -0.2586],\n        [-0.8476, -0.0004, -1.1003,  2.2806, -1.2226,  0.9251, -0.3165,\n          0.4957,  0.0690,  0.0232],\n        [-0.9108,  1.1355, -0.2715,  0.2233, -0.3681,  0.1442, -0.0001,\n         -0.0174,  0.1454,  0.2286],\n        [-1.0663, -0.8466, -0.7147,  2.5685, -0.2090,  1.2993, -0.3057,\n         -0.8314,  0.7046, -0.0176],\n        [ 1.7013, -1.8051,  0.7541, -1.5248,  0.8972,  0.1518,  1.4876,\n         -0.8454, -0.2022, -0.2829],\n        [-0.8179, -0.1239,  0.8630, -0.2137, -0.2275, -0.5411, -1.3448,\n          1.7354,  0.7751,  0.6234],\n        [ 0.6515, -1.0431,  2.7165,  0.1873, -1.0623,  0.1286,  0.3597,\n         -0.2739,  0.3871, -1.6699],\n        [-0.2828, -1.4663,  0.1182, -0.0896, -0.3640, -0.5129, -0.4905,\n          2.2914, -0.2227,  0.9463],\n        [-1.2596,  2.0468, -0.4405, -0.0411, -0.8073,  0.0490, -0.0604,\n         -0.1206,  0.3504, -0.1059],\n        [ 0.6089,  0.5885,  0.7898,  1.1318, -1.9008,  0.5875,  0.4227,\n         -1.1815,  0.5652, -1.3590],\n        [-1.4551,  2.9537, -0.2805,  0.2372, -1.4180,  0.0297, -0.1515,\n         -0.6111,  0.6140, -0.3354],\n        [-0.7182,  1.6778,  0.0553,  0.0461, -0.5446, -0.0338, -0.0215,\n         -0.0881,  0.1506, -0.2107],\n        [-0.8027, -0.7854, -0.1275, -0.3177, -0.1600, -0.1964, -0.6084,\n          2.1285, -0.1815,  1.1911],\n        [-2.0656, -0.4959, -0.1154, -0.1363,  2.2426, -0.7441, -0.8413,\n          0.4675,  0.3269,  1.7279],\n        [-0.3004,  1.0166,  1.1175, -0.0618, -0.0937, -0.4221,  0.1943,\n         -1.1020,  0.3670, -0.4683],\n        [-1.0720,  0.2252,  0.0175,  1.3644, -0.7409,  0.4655,  0.5439,\n          0.0380,  0.1279, -0.2302],\n        [ 0.2409, -1.2622, -0.6336,  1.8240, -0.5951,  1.3408,  0.2130,\n         -1.3789,  0.8363, -0.2101],\n        [-1.3849,  0.3773, -0.0585,  0.6896, -0.0998,  0.2804,  0.0696,\n         -0.2529,  0.3143,  0.3409],\n        [-0.9103, -0.1578,  1.6673, -0.4817,  0.4088, -0.5484,  0.6103,\n         -0.2287, -0.0665,  0.0055],\n        [-1.1692, -2.8531, -1.2499, -0.0257,  2.8580,  0.2616, -0.7122,\n         -0.0551,  0.8112,  2.3233],\n        [-0.2790, -1.9494,  0.6096, -0.5653,  2.2792, -1.0687,  0.1634,\n          0.3122,  0.1053,  1.0884],\n        [ 0.1267, -1.2297, -0.1315,  0.2428, -0.5436,  0.4123,  2.3060,\n         -0.9278, -0.1528, -0.4224],\n        [-0.0235, -0.9137, -0.1457,  1.6858, -0.7552,  0.7293,  0.2510,\n         -0.3955, -0.2187, -0.1505],\n        [ 0.5643, -1.2783, -1.4149,  0.0304,  0.8375,  1.5018,  0.0338,\n         -0.3875, -0.0117,  0.5751],\n        [ 0.2926, -0.7486, -0.3238,  1.0384,  0.0308,  0.6792, -0.0170,\n         -0.5797,  0.2819, -0.3510],\n        [ 0.1219, -0.5862,  1.5817, -0.1297,  0.4730, -0.9171,  0.7886,\n         -0.7022, -0.0501, -0.2812],\n        [ 1.7587, -2.4511, -0.7369,  0.4082, -0.6426,  1.1784,  0.6052,\n         -0.7178,  1.6161, -0.2220],\n        [-0.1267, -2.6719,  0.0505, -0.4972,  2.9027, -0.1461,  0.2807,\n         -0.2921,  0.2231,  1.1327],\n        [-0.9892,  2.4401,  0.1274,  0.2838, -0.7535, -0.1684, -0.6493,\n         -0.1908,  0.2290, -0.2150],\n        [-0.2071, -2.1351, -0.9191, -0.9309,  1.7747, -0.3046,  0.0183,\n          1.0136, -0.1016,  2.1288],\n        [-0.0103,  0.3280, -0.6974, -0.2504,  0.3187,  0.4390, -0.1879,\n          0.3954,  0.2332, -0.1971],\n        [-0.2280, -1.6754, -0.7438,  0.5078,  0.2544, -0.1020, -0.2503,\n          2.0799, -0.5033,  0.5890],\n        [ 0.3972, -0.9369,  1.2696, -1.6713, -0.4159, -0.0221,  0.6489,\n         -0.4777,  1.2497,  0.3931],\n        [-0.7566, -0.8230, -0.0785, -0.3083,  0.7821,  0.1880,  0.1037,\n         -0.0956,  0.4219,  1.0798],\n        [-1.0328, -0.1700,  1.3806,  0.5445, -0.2624, -0.0780, -0.3595,\n         -0.6253,  0.4309,  0.1813],\n        [-1.0360, -0.4704,  0.1948, -0.7066,  0.6600, -0.4633, -0.3602,\n          1.7494,  0.1522,  0.6086],\n        [-1.2032, -0.7903, -0.5754,  0.4722,  0.6068,  0.5752,  0.2151,\n         -0.2495,  0.3420,  0.9278],\n        [ 0.2247, -0.1361,  0.9374, -0.1543,  0.4921, -0.6553,  0.5885,\n          0.2617, -0.2216, -0.3736],\n        [-0.2867, -1.4486,  0.6658, -0.8755,  2.3195, -0.7627, -0.2132,\n          0.2488,  0.3484,  1.0860],\n        [-1.4031, -0.4518, -0.3181,  2.8268, -0.5371,  1.0154, -0.9247,\n         -0.7385,  1.1031,  0.0422],\n        [ 2.8604, -1.5413,  0.6241, -0.8017, -1.4104,  0.6314,  0.4614,\n         -0.0218, -0.3411, -0.2609],\n        [ 0.2113, -1.2348, -0.8535, -0.1041, -0.2703, -0.1294, -0.7057,\n          2.7552, -0.4429,  0.4517],\n        [ 4.5191, -2.7407,  1.1091,  0.3975, -0.9456,  1.2277,  0.3616,\n         -1.6564,  0.5063, -1.4274],\n        [ 1.4615, -1.0765,  1.8388,  1.5006, -1.2351,  0.2781,  0.2830,\n         -0.8491,  0.2222, -1.7779],\n        [-1.2160,  0.8502,  0.2413, -0.0798, -0.7880, -0.4286, -0.8060,\n          0.7194,  1.2663,  0.6412],\n        [-1.3318,  2.3388, -0.4003, -0.1094, -1.0285,  0.1021, -0.0388,\n         -0.0497,  0.5137, -0.2507],\n        [-1.7853,  0.5884, -0.6108, -0.5557,  0.8696, -0.6226, -0.7983,\n          1.7169, -0.0145,  0.8231],\n        [-0.1739,  0.1562, -0.2933,  2.3195, -0.9480,  1.2019, -0.4834,\n         -1.0567,  0.5685, -0.6841],\n        [-0.7920, -0.3339,  0.7452, -0.6529, -0.3307, -0.6092, -0.0950,\n          1.7311, -0.3481,  0.3801],\n        [-1.7810,  1.0676, -0.7611,  0.3658, -0.0431, -0.1012, -0.6048,\n          0.3089,  0.9998,  0.7164],\n        [-0.5856, -0.5261, -0.4859, -1.0551, -0.1838, -0.2144, -1.2599,\n          3.3891,  0.4691,  0.7566],\n        [-0.4984, -1.7770, -1.1998, -0.1075,  1.0882,  0.4539, -0.5651,\n          1.4381, -0.5678,  1.7479],\n        [ 0.2938, -1.8536,  0.4259, -0.5429,  0.0066,  0.4120,  2.3793,\n         -0.3666, -0.2604,  0.0382],\n        [-0.4080, -0.9851,  4.0264,  0.1099, -0.1766, -1.1557,  0.6419,\n         -0.8147,  0.7535, -1.1452],\n        [-0.4636, -1.7323, -0.6433, -0.0274,  0.7227, -0.1799, -0.9336,\n          2.1881, -0.2073,  1.6522],\n        [-0.9617, -0.0348, -0.3980, -0.4738,  0.7790,  0.4671, -0.6115,\n         -0.7067,  1.3036,  0.4923],\n        [-1.0151, -2.5385, -0.6072,  0.2902,  3.1570,  0.1062, -0.2169,\n         -0.4491,  0.6326,  1.6829],\n        [-1.8852,  0.6066, -0.2840, -0.4475, -0.1147, -0.7858, -1.1805,\n          3.0723,  0.3960,  0.9720],\n        [ 0.0344, -1.4878, -0.9675,  1.9649, -0.3146,  1.2183,  0.6730,\n         -0.3650,  0.0646, -0.0898],\n        [-0.2118, -2.0350,  0.9917, -0.8993,  1.2334, -0.6723,  2.5847,\n         -0.0454, -0.4149,  0.3927],\n        [-1.7365,  3.0447,  0.5115,  0.0786, -0.7544, -0.2158, -0.4876,\n         -0.2891,  0.5089, -0.6719],\n        [ 0.3652, -0.5457, -0.1167,  2.9056, -1.1622,  0.8192, -1.3245,\n         -0.6414,  0.8097, -0.4958],\n        [-0.8755, -0.6983,  0.2208, -0.6463,  0.5276,  0.1145,  2.7229,\n         -1.0316,  0.1905,  0.2090],\n        [-0.9702,  0.1265, -0.0007, -0.5106,  0.4970, -0.0804,  0.0017,\n          0.0607,  0.6164,  0.4490],\n        [-0.8271, -0.6822, -0.7434,  2.6457, -1.6143,  1.1486, -1.0705,\n          0.5611,  0.6422,  0.1250],\n        [-1.9979,  1.8175, -0.1658, -0.0343, -0.6292,  0.1774,  0.3150,\n         -0.4633,  0.9266,  0.0252],\n        [-0.9039, -0.6030, -0.2173, -1.1768,  2.3198, -0.5072,  0.3418,\n         -0.1551,  0.1282,  1.4250],\n        [-0.9891,  0.5212, -0.4518,  0.3267, -0.0759,  0.3826, -0.0341,\n          0.0382,  0.2451,  0.3658],\n        [-2.1217,  1.5102, -0.7828,  0.3554, -0.4192, -0.0772,  0.0578,\n          0.8070,  0.1701,  0.5880],\n        [ 1.0665, -1.3826,  0.6243, -0.8096, -0.4227,  0.5925,  1.8112,\n         -0.9946,  0.2010, -0.7731],\n        [-1.1263, -1.7484,  0.0041, -0.5439,  1.7242, -0.9475, -0.3835,\n          0.8452,  0.3077,  2.2689]])\n</code></pre> <p>Printing output size</p> <p>This produces a 100x10 matrix because each iteration has a batch size of 100 and each prediction across the 10 classes, with the largest number indicating the likely number it is predicting. <pre><code>iter_test = 0\nfor images, labels in test_loader:\n    iter_test += 1\n    images = images.view(-1, 28*28).requires_grad_()\n    outputs = model(images)\n    if iter_test == 1:\n        print('OUTPUTS')\n        print(outputs.size())\n    _, predicted = torch.max(outputs.data, 1)\n</code></pre></p> <pre><code>OUTPUTS\ntorch.Size([100, 10])\n</code></pre> <p>Printing one output</p> <p>This would be a 1x10 matrix where the largest number is what the model thinks the image is. Here we can see that in the tensor, position 7 has the largest number, indicating the model thinks the image is 7.</p> <p>number 0: -0.4181  number 1: -1.0784 ...  number 7: 2.9352 <pre><code>iter_test = 0\nfor images, labels in test_loader:\n    iter_test += 1\n    images = images.view(-1, 28*28).requires_grad_()\n    outputs = model(images)\n    if iter_test == 1:\n        print('OUTPUTS')\n        print(outputs[0, :])\n    _, predicted = torch.max(outputs.data, 1)\n</code></pre></p> <pre><code>OUTPUTS\ntensor([-0.4181, -1.0784, -0.4840, -0.0985, -0.2394, -0.1801, -1.1639,\n         2.9352, -0.1552,  0.8852])\n</code></pre> <p>Printing prediction output</p> <p>Because our output is of size 100 (our batch size), our prediction size would also of the size 100.</p> <pre><code>iter_test = 0\nfor images, labels in test_loader:\n    iter_test += 1\n    images = images.view(-1, 28*28).requires_grad_()\n    outputs = model(images)\n    _, predicted = torch.max(outputs.data, 1)\n    if iter_test == 1:\n        print('PREDICTION')\n        print(predicted.size())\n</code></pre> <pre><code>PREDICTION\ntorch.Size([100])\n</code></pre> <p>Print prediction value</p> <p>We are printing our prediction which as verified above, should be digit 7.</p> <pre><code>iter_test = 0\nfor images, labels in test_loader:\n    iter_test += 1\n    images = images.view(-1, 28*28).requires_grad_()\n    outputs = model(images)\n    _, predicted = torch.max(outputs.data, 1)\n    if iter_test == 1:\n        print('PREDICTION')\n        print(predicted[0])\n</code></pre> <pre><code>PREDICTION\ntensor(7)\n</code></pre> <p>Print prediction, label and label size</p> <p>We are trying to show what we are predicting and the actual values. In this case, we're predicting the right value 7!</p> <pre><code>iter_test = 0\nfor images, labels in test_loader:\n    iter_test += 1\n    images = images.view(-1, 28*28).requires_grad_()\n    outputs = model(images)\n    _, predicted = torch.max(outputs.data, 1)\n    if iter_test == 1:\n        print('PREDICTION')\n        print(predicted[0])\n\n        print('LABEL SIZE')\n        print(labels.size())\n\n        print('LABEL FOR IMAGE 0')\n        print(labels[0])\n</code></pre> <pre><code>PREDICTION\ntensor(7)\n\nLABEL SIZE\ntorch.Size([100])\n\nLABEL FOR IMAGE 0\ntensor(7)\n</code></pre> <p>Print second prediction and ground truth</p> <p>Again, the prediction is correct. Naturally, as our model is quite competent in this simple task.</p> <pre><code>iter_test = 0\nfor images, labels in test_loader:\n    iter_test += 1\n    images = images.view(-1, 28*28).requires_grad_()\n    outputs = model(images)\n    _, predicted = torch.max(outputs.data, 1)\n\n    if iter_test == 1:\n        print('PREDICTION')\n        print(predicted[1])\n\n        print('LABEL SIZE')\n        print(labels.size())\n\n        print('LABEL FOR IMAGE 1')\n        print(labels[1])\n</code></pre> <pre><code>PREDICTION\ntensor(2)\n\nLABEL SIZE\ntorch.Size([100])\n\nLABEL FOR IMAGE 1\ntensor(2)\n</code></pre> <p>Print accuracy</p> <p>Now we know what each object represents, we can understand how we arrived at our accuracy numbers.</p> <p>One last thing to note is that <code>correct.item()</code> has this syntax is because <code>correct</code> is a PyTorch tensor and to get the value to compute with <code>total</code> which is an integer, we need to do this. <pre><code>correct = 0\ntotal = 0\niter_test = 0\nfor images, labels in test_loader:\n    iter_test += 1\n    images = images.view(-1, 28*28).requires_grad_()\n    outputs = model(images)\n    _, predicted = torch.max(outputs.data, 1)\n\n    # Total number of labels\n    total += labels.size(0)\n\n    # Total correct predictions\n    correct += (predicted == labels).sum()\n\naccuracy = 100 * (correct.item() / total)\n\nprint(accuracy)\n</code></pre></p> <pre><code>82.94\n</code></pre> <p>Explanation of Python's .sum() function</p> <p>Python's .sum() function allows you to do a comparison between two matrices and sum the ones that return <code>True</code> or in our case, those predictions that match actual labels (correct predictions).</p> <pre><code># Explaining .sum() python built-in function\n# correct += (predicted == labels).sum()\nimport numpy as np\na = np.ones((10))\nprint(a)\nb = np.ones((10))\nprint(b)\n\nprint(a == b)\n\nprint((a == b).sum())\n</code></pre> <pre><code># matrix a\n[1. 1. 1. 1. 1. 1. 1. 1. 1. 1.]\n\n# matrix b\n[1. 1. 1. 1. 1. 1. 1. 1. 1. 1.]\n\n# boolean array\n[ True  True  True  True  True  True  True  True  True  True]\n\n# number of elementswhere a matches b\n10\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#saving-model","title":"Saving Model","text":"<p>Saving PyTorch model</p> <p>This is how you save your model. Feel free to just change <code>save_model = True</code> to save your model <pre><code>save_model = False\nif save_model is True:\n    # Saves only parameters\n    torch.save(model.state_dict(), 'awesome_model.pkl')\n</code></pre></p>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#building-a-logistic-regression-model-with-pytorch-gpu","title":"Building a Logistic Regression Model with PyTorch (GPU)","text":"<p>CPU version</p> <p>The usual 7-step process, getting repetitive by now which we like. </p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass LogisticRegressionModel(nn.Module):\n    def __init__(self, input_size, num_classes):\n        super(LogisticRegressionModel, self).__init__()\n        self.linear = nn.Linear(input_dim, output_dim)\n\n    def forward(self, x):\n        out = self.linear(x)\n        return out\n\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\noutput_dim = 10\n\nmodel = LogisticRegressionModel(input_dim, output_dim)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.001\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as Variable\n        images = images.view(-1, 28*28).requires_grad_()\n        labels = labels\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        # 100 x 10\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, 28*28).requires_grad_()\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                # 100 x 1\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct.item() / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 1.876196026802063. Accuracy: 64.44\nIteration: 1000. Loss: 1.5153584480285645. Accuracy: 75.68\nIteration: 1500. Loss: 1.3521136045455933. Accuracy: 78.98\nIteration: 2000. Loss: 1.2136967182159424. Accuracy: 80.95\nIteration: 2500. Loss: 1.0934826135635376. Accuracy: 81.97\nIteration: 3000. Loss: 1.024120569229126. Accuracy: 82.49\n</code></pre> <p>GPU version</p> <p>2 things must be on GPU - <code>model</code> - <code>tensors</code></p> <p>Remember step 4 and 7 will be affected and this will be the same for all model building moving forward.</p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n'''\nSTEP 1: LOADING DATASET\n'''\n\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\nclass LogisticRegressionModel(nn.Module):\n    def __init__(self, input_size, num_classes):\n        super(LogisticRegressionModel, self).__init__()\n        self.linear = nn.Linear(input_dim, output_dim)\n\n    def forward(self, x):\n        out = self.linear(x)\n        return out\n\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28*28\noutput_dim = 10\n\nmodel = LogisticRegressionModel(input_dim, output_dim)\n\n#######################\n#  USE GPU FOR MODEL  #\n#######################\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.001\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n\n        #######################\n        #  USE GPU FOR MODEL  #\n        #######################\n        images = images.view(-1, 28*28).requires_grad_().to(device)\n        labels = labels.to(device)\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                #######################\n                #  USE GPU FOR MODEL  #\n                #######################\n                images = images.view(-1, 28*28).to(device)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                #######################\n                #  USE GPU FOR MODEL  #\n                #######################\n                # Total correct predictions\n                if torch.cuda.is_available():\n                    correct += (predicted.cpu() == labels.cpu()).sum()\n                else:\n                    correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct.item() / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 1.8571407794952393. Accuracy: 68.99\nIteration: 1000. Loss: 1.5415704250335693. Accuracy: 75.86\nIteration: 1500. Loss: 1.2755383253097534. Accuracy: 78.92\nIteration: 2000. Loss: 1.2468739748001099. Accuracy: 80.72\nIteration: 2500. Loss: 1.0708973407745361. Accuracy: 81.73\nIteration: 3000. Loss: 1.0359245538711548. Accuracy: 82.74\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#summary","title":"Summary","text":"<p>We've learnt to...</p> <p>Success</p> <ul> <li> Logistic regression basics</li> <li> Problems of linear regression</li> <li> In-depth Logistic Regression<ul> <li> Get logits</li> <li> Get softmax</li> <li> Get cross-entropy loss</li> </ul> </li> <li> Aim: reduce cross-entropy loss</li> <li> Built a logistic regression model in CPU and GPU<ul> <li> Step 1: Load Dataset</li> <li> Step 2: Make Dataset Iterable</li> <li> Step 3: Create Model Class</li> <li> Step 4: Instantiate Model Class</li> <li> Step 5: Instantiate Loss Class</li> <li> Step 6: Instantiate Optimizer Class</li> <li> Step 7: Train Model</li> </ul> </li> <li> Important things to be on GPU<ul> <li> <code>model</code></li> <li> <code>tensors with gradients</code></li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_logistic_regression/#citation","title":"Citation","text":"<p>If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.</p> <p></p>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/","title":"Long Short-Term Memory (LSTM) network with PyTorch","text":"<p>Run Jupyter Notebook</p> <p>You can run the code for this section in this jupyter notebook link.</p>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#about-lstms-special-rnn","title":"About LSTMs: Special RNN","text":"<ul> <li>Capable of learning long-term dependencies</li> <li>LSTM = RNN on super juice</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#rnn-transition-to-lstm","title":"RNN Transition to LSTM","text":""},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#building-an-lstm-with-pytorch","title":"Building an LSTM with PyTorch","text":""},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#model-a-1-hidden-layer","title":"Model A: 1 Hidden Layer","text":"<ul> <li>Unroll 28 time steps<ul> <li>Each step input size: 28 x 1</li> <li>Total per unroll: 28 x 28<ul> <li>Feedforward Neural Network input size: 28 x 28 </li> </ul> </li> </ul> </li> <li>1 Hidden layer</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#steps","title":"Steps","text":"<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#step-1-loading-mnist-train-dataset","title":"Step 1: Loading MNIST Train Dataset","text":"<p>Images from 1 to 9</p> <p>The usual loading of our MNIST dataset</p> <p>As usual, we've 60k training images and 10k testing images. </p> <p>Subsequently, we'll have 3 groups: training, validation and testing for a more robust evaluation of algorithms.</p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n</code></pre> <pre><code>train_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n</code></pre> <pre><code>print(train_dataset.train_data.size())\n</code></pre> <pre><code>print(train_dataset.train_labels.size())\n</code></pre> <pre><code>print(test_dataset.test_data.size())\n\n\n```python\nprint(test_dataset.test_labels.size())\n</code></pre> <pre><code>torch.Size([60000, 28, 28])\ntorch.Size([60000])\n\ntorch.Size([10000, 28, 28])\ntorch.Size([10000])\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#step-2-make-dataset-iterable","title":"Step 2: Make Dataset Iterable","text":"<p>Creating an iterable object for our dataset</p> <pre><code>batch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#step-3-create-model-class","title":"Step 3: Create Model Class","text":"<p>Creating an LSTM model class</p> <p>It is very similar to RNN in terms of the shape of our input of <code>batch_dim x seq_dim x feature_dim</code>.</p> <p>The only change is that we have our cell state on top of our hidden state. PyTorch's LSTM module handles all the other weights for our other gates.</p> <pre><code>class LSTMModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):\n        super(LSTMModel, self).__init__()\n        # Hidden dimensions\n        self.hidden_dim = hidden_dim\n\n        # Number of hidden layers\n        self.layer_dim = layer_dim\n\n        # Building your LSTM\n        # batch_first=True causes input/output tensors to be of shape\n        # (batch_dim, seq_dim, feature_dim)\n        self.lstm = nn.LSTM(input_dim, hidden_dim, layer_dim, batch_first=True)\n\n        # Readout layer\n        self.fc = nn.Linear(hidden_dim, output_dim)\n\n    def forward(self, x):\n        # Initialize hidden state with zeros\n        h0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).requires_grad_()\n\n        # Initialize cell state\n        c0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).requires_grad_()\n\n        # 28 time steps\n        # We need to detach as we are doing truncated backpropagation through time (BPTT)\n        # If we don't, we'll backprop all the way to the start even after going through another batch\n        out, (hn, cn) = self.lstm(x, (h0.detach(), c0.detach()))\n\n        # Index hidden state of last time step\n        # out.size() --&gt; 100, 28, 100\n        # out[:, -1, :] --&gt; 100, 100 --&gt; just want last time step hidden states! \n        out = self.fc(out[:, -1, :]) \n        # out.size() --&gt; 100, 10\n        return out\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#step-4-instantiate-model-class","title":"Step 4: Instantiate Model Class","text":"<ul> <li>28 time steps<ul> <li>Each time step: input dimension = 28</li> </ul> </li> <li>1 hidden layer</li> <li>MNIST 1-9 digits \\(\\rightarrow\\) output dimension = 10</li> </ul> <p>Instantiate our LSTM model</p> <pre><code>input_dim = 28\nhidden_dim = 100\nlayer_dim = 1\noutput_dim = 10\n</code></pre> <pre><code>model = LSTMModel(input_dim, hidden_dim, layer_dim, output_dim)\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#step-5-instantiate-loss-class","title":"Step 5: Instantiate Loss Class","text":"<ul> <li>Long Short-Term Memory Neural Network: Cross Entropy Loss<ul> <li>Recurrent Neural Network: Cross Entropy Loss</li> <li>Convolutional Neural Network: Cross Entropy Loss</li> <li>Feedforward Neural Network: Cross Entropy Loss</li> <li>Logistic Regression: Cross Entropy Loss</li> <li>Linear Regression: MSE</li> </ul> </li> </ul> <p>Cross Entry Loss Function</p> <p>Because we are doing a classification problem we'll be using a Cross Entropy function. If we were to do a regression problem, then we would typically use a MSE function.</p> <pre><code>criterion = nn.CrossEntropyLoss()\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#step-6-instantiate-optimizer-class","title":"Step 6: Instantiate Optimizer Class","text":"<ul> <li>Simplified equation<ul> <li>\\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\)<ul> <li>\\(\\theta\\): parameters (our variables)</li> <li>\\(\\eta\\): learning rate (how fast we want to learn)</li> <li>\\(\\nabla_\\theta\\): parameters' gradients</li> </ul> </li> </ul> </li> <li>Even simplier equation<ul> <li><code>parameters = parameters - learning_rate * parameters_gradients</code></li> <li>At every iteration, we update our model's parameters</li> </ul> </li> </ul> <p>Mini-batch Stochastic Gradient Descent</p> <pre><code>learning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)  \n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#parameters-in-depth","title":"Parameters In-Depth","text":"<p>1 Layer LSTM Groups of Parameters</p> <p>We will have 6 groups of parameters here comprising weights and biases from:     - Input to Hidden Layer Affine Function     - Hidden Layer to Output Affine Function     - Hidden Layer to Hidden Layer Affine Function</p> <p>Notice how this is exactly the same number of groups of parameters as our RNN? But the sizes of these groups will be larger for an LSTM due to its gates.</p> <pre><code>len(list(model.parameters()))\n</code></pre> <pre><code>6\n</code></pre> <p>In-depth Parameters Analysis</p> <p>Comparing to RNN's parameters, we've the same number of groups but for LSTM we've 4x the number of parameters!</p> <pre><code>for i in range(len(list(model.parameters()))):\n    print(list(model.parameters())[i].size())\n</code></pre> <pre><code>torch.Size([400, 28])\ntorch.Size([400, 100])\n\ntorch.Size([400])\ntorch.Size([400])\n\ntorch.Size([10, 100])\ntorch.Size([10])\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#parameters-breakdown","title":"Parameters Breakdown","text":"<ul> <li>This is the breakdown of the parameters associated with the respective affine functions </li> <li>Input \\(\\rightarrow\\) Gates<ul> <li>\\([400, 28] \\rightarrow w_1, w_3, w_5, w_7\\)</li> <li>\\([400] \\rightarrow b_1, b_3, b_5, b_7\\)</li> </ul> </li> <li>Hidden State \\(\\rightarrow\\) Gates<ul> <li>\\([400,100] \\rightarrow w_2, w_4, w_6, w_8\\)</li> <li>\\([400] \\rightarrow b_2, b_4, b_6, b_8\\)</li> </ul> </li> <li>Hidden State \\(\\rightarrow\\) Output<ul> <li>\\([10, 100] \\rightarrow w_9\\)</li> <li>\\([10] \\rightarrow b_9\\)</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#step-7-train-model","title":"Step 7: Train Model","text":"<ul> <li>Process <ol> <li>Convert inputs/labels to variables<ul> <li>LSTM Input: (1, 28)</li> <li>RNN Input: (1, 28)</li> <li>CNN Input: (1, 28, 28) </li> <li>FNN Input: (1, 28*28)</li> </ul> </li> <li>Clear gradient buffets</li> <li>Get output given inputs </li> <li>Get loss</li> <li>Get gradients w.r.t. parameters</li> <li>Update parameters using gradients<ul> <li><code>parameters = parameters - learning_rate * parameters_gradients</code></li> </ul> </li> <li>REPEAT</li> </ol> </li> </ul> <p>Training 1 Hidden Layer LSTM</p> <pre><code># Number of steps to unroll\nseq_dim = 28  \n\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as a torch tensor with gradient accumulation abilities\n        images = images.view(-1, seq_dim, input_dim).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        # outputs.size() --&gt; 100, 10\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Resize images\n                images = images.view(-1, seq_dim, input_dim)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 0.8390830755233765. Accuracy: 72\nIteration: 1000. Loss: 0.46470555663108826. Accuracy: 85\nIteration: 1500. Loss: 0.31465113162994385. Accuracy: 91\nIteration: 2000. Loss: 0.19143860042095184. Accuracy: 94\nIteration: 2500. Loss: 0.16134005784988403. Accuracy: 95\nIteration: 3000. Loss: 0.255976140499115. Accuracy: 95\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#model-b-2-hidden-layer","title":"Model B: 2 Hidden Layer","text":"<ul> <li>Unroll 28 time steps<ul> <li>Each step input size: 28 x 1</li> <li>Total per unroll: 28 x 28<ul> <li>Feedforward Neural Network inpt size: 28 x 28 </li> </ul> </li> </ul> </li> <li>2 Hidden layer</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#steps_1","title":"Steps","text":"<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> </ul> <p>Train 2 Hidden Layer LSTM</p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n'''\nSTEP 1: LOADING DATASET\n'''\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\n\nclass LSTMModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):\n        super(LSTMModel, self).__init__()\n        # Hidden dimensions\n        self.hidden_dim = hidden_dim\n\n        # Number of hidden layers\n        self.layer_dim = layer_dim\n\n        # Building your LSTM\n        # batch_first=True causes input/output tensors to be of shape\n        # (batch_dim, seq_dim, feature_dim)\n        self.lstm = nn.LSTM(input_dim, hidden_dim, layer_dim, batch_first=True)\n\n        # Readout layer\n        self.fc = nn.Linear(hidden_dim, output_dim)\n\n    def forward(self, x):\n        # Initialize hidden state with zeros\n        h0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).requires_grad_()\n\n        # Initialize cell state\n        c0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).requires_grad_()\n\n        # One time step\n        # We need to detach as we are doing truncated backpropagation through time (BPTT)\n        # If we don't, we'll backprop all the way to the start even after going through another batch\n        out, (hn, cn) = self.lstm(x, (h0.detach(), c0.detach()))\n\n        # Index hidden state of last time step\n        # out.size() --&gt; 100, 28, 100\n        # out[:, -1, :] --&gt; 100, 100 --&gt; just want last time step hidden states! \n        out = self.fc(out[:, -1, :]) \n        # out.size() --&gt; 100, 10\n        return out\n\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28\nhidden_dim = 100\nlayer_dim = 2  # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER\noutput_dim = 10\n\nmodel = LSTMModel(input_dim, hidden_dim, layer_dim, output_dim)\n\n# JUST PRINTING MODEL &amp; PARAMETERS \nprint(model)\nprint(len(list(model.parameters())))\nfor i in range(len(list(model.parameters()))):\n    print(list(model.parameters())[i].size())\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)  \n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\n\n# Number of steps to unroll\nseq_dim = 28  \n\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as torch tensor with gradient accumulation abilities\n        images = images.view(-1, seq_dim, input_dim).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        # outputs.size() --&gt; 100, 10\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Resize image\n                images = images.view(-1, seq_dim, input_dim)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>LSTMModel(\n  (lstm): LSTM(28, 100, num_layers=2, batch_first=True)\n  (fc): Linear(in_features=100, out_features=10, bias=True)\n)\n\n10\n\ntorch.Size([400, 28])\ntorch.Size([400, 100])\ntorch.Size([400])\ntorch.Size([400])\ntorch.Size([400, 100])\ntorch.Size([400, 100])\ntorch.Size([400])\ntorch.Size([400])\ntorch.Size([10, 100])\ntorch.Size([10])\n\nIteration: 500. Loss: 2.3074915409088135. Accuracy: 11\nIteration: 1000. Loss: 1.8854578733444214. Accuracy: 35\nIteration: 1500. Loss: 0.5317062139511108. Accuracy: 80\nIteration: 2000. Loss: 0.15290376543998718. Accuracy: 92\nIteration: 2500. Loss: 0.19500978291034698. Accuracy: 93\nIteration: 3000. Loss: 0.10683634132146835. Accuracy: 95\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#parameters-breakdown-layer-1","title":"Parameters Breakdown (Layer 1)","text":"<ul> <li>Input \\(\\rightarrow\\) Gates<ul> <li>\\([400, 28]\\)</li> <li>\\([400]\\)</li> </ul> </li> <li>Hidden State \\(\\rightarrow\\) Gates<ul> <li>\\([400,100]\\)</li> <li>\\([400]\\)</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#parameters-breakdown-layer-2","title":"Parameters Breakdown (Layer 2)","text":"<ul> <li>Input \\(\\rightarrow\\) Gates <ul> <li>\\([400, 100]\\)</li> <li>\\([400]\\)</li> </ul> </li> <li>Hidden State \\(\\rightarrow\\) Gates <ul> <li>\\([400,100]\\)</li> <li>\\([400]\\)</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#parameters-breakdown-readout-layer","title":"Parameters Breakdown (Readout Layer)","text":"<ul> <li>Hidden State \\(\\rightarrow\\) Output<ul> <li>\\([10, 100]\\)</li> <li>\\([10]\\)</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#model-c-3-hidden-layer","title":"Model C: 3 Hidden Layer","text":"<ul> <li>Unroll 28 time steps<ul> <li>Each step input size: 28 x 1</li> <li>Total per unroll: 28 x 28<ul> <li>Feedforward Neural Network inpt size: 28 x 28 </li> </ul> </li> </ul> </li> <li>3 Hidden layer</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#steps_2","title":"Steps","text":"<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> </ul> <p>3 Hidden Layer LSTM</p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n'''\nSTEP 1: LOADING DATASET\n'''\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\n\nclass LSTMModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):\n        super(LSTMModel, self).__init__()\n        # Hidden dimensions\n        self.hidden_dim = hidden_dim\n\n        # Number of hidden layers\n        self.layer_dim = layer_dim\n\n        # Building your LSTM\n        # batch_first=True causes input/output tensors to be of shape\n        # (batch_dim, seq_dim, feature_dim)\n        self.lstm = nn.LSTM(input_dim, hidden_dim, layer_dim, batch_first=True)\n\n        # Readout layer\n        self.fc = nn.Linear(hidden_dim, output_dim)\n\n    def forward(self, x):\n        # Initialize hidden state with zeros\n        h0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).requires_grad_()\n\n        # Initialize cell state\n        c0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).requires_grad_()\n\n        # One time step\n        # We need to detach as we are doing truncated backpropagation through time (BPTT)\n        # If we don't, we'll backprop all the way to the start even after going through another batch\n        out, (hn, cn) = self.lstm(x, (h0.detach(), c0.detach()))\n\n        # Index hidden state of last time step\n        # out.size() --&gt; 100, 28, 100\n        # out[:, -1, :] --&gt; 100, 100 --&gt; just want last time step hidden states! \n        out = self.fc(out[:, -1, :]) \n        # out.size() --&gt; 100, 10\n        return out\n\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28\nhidden_dim = 100\nlayer_dim = 3  # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER\noutput_dim = 10\n\nmodel = LSTMModel(input_dim, hidden_dim, layer_dim, output_dim)\n\n# JUST PRINTING MODEL &amp; PARAMETERS \nprint(model)\nprint(len(list(model.parameters())))\nfor i in range(len(list(model.parameters()))):\n    print(list(model.parameters())[i].size())\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)  \n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\n\n# Number of steps to unroll\nseq_dim = 28  \n\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as Variable\n        images = images.view(-1, seq_dim, input_dim).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        # outputs.size() --&gt; 100, 10\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch Variable\n                images = images.view(-1, seq_dim, input_dim).requires_grad_()\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>LSTMModel(\n  (lstm): LSTM(28, 100, num_layers=3, batch_first=True)\n  (fc): Linear(in_features=100, out_features=10, bias=True)\n)\n\n14\n\ntorch.Size([400, 28])\ntorch.Size([400, 100])\ntorch.Size([400])\ntorch.Size([400])\ntorch.Size([400, 100])\ntorch.Size([400, 100])\ntorch.Size([400])\ntorch.Size([400])\ntorch.Size([400, 100])\ntorch.Size([400, 100])\ntorch.Size([400])\ntorch.Size([400])\ntorch.Size([10, 100])\ntorch.Size([10])\n\nIteration: 500. Loss: 2.2927396297454834. Accuracy: 11\nIteration: 1000. Loss: 2.29740309715271. Accuracy: 11\nIteration: 1500. Loss: 2.1950502395629883. Accuracy: 20\nIteration: 2000. Loss: 1.0738657712936401. Accuracy: 59\nIteration: 2500. Loss: 0.5988132357597351. Accuracy: 79\nIteration: 3000. Loss: 0.4107239246368408. Accuracy: 88\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#parameters-breakdown-layer-1_1","title":"Parameters Breakdown (Layer 1)","text":"<ul> <li>Input \\(\\rightarrow\\) Gates<ul> <li>[400, 28]</li> <li>[400]</li> </ul> </li> <li>Hidden State \\(\\rightarrow\\) Gates<ul> <li>[400,100]</li> <li>[400]</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#parameters-breakdown-layer-2_1","title":"Parameters Breakdown (Layer 2)","text":"<ul> <li>Input \\(\\rightarrow\\) Gates <ul> <li>[400, 100]</li> <li>[400]</li> </ul> </li> <li>Hidden State \\(\\rightarrow\\) Gates <ul> <li>[400,100]</li> <li>[400]</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#parameters-breakdown-layer-3","title":"Parameters Breakdown (Layer 3)","text":"<ul> <li>Input \\(\\rightarrow\\) Gates <ul> <li>[400, 100]</li> <li>[400]</li> </ul> </li> <li>Hidden State \\(\\rightarrow\\) Gates <ul> <li>[400,100]</li> <li>[400]</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#parameters-breakdown-readout-layer_1","title":"Parameters Breakdown (Readout Layer)","text":"<ul> <li>Hidden State \\(\\rightarrow\\) Output<ul> <li>[10, 100]</li> <li>[10]</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#comparison-with-rnn","title":"Comparison with RNN","text":"Model A RNN Model B RNN Model C RNN ReLU ReLU Tanh 1 Hidden Layer 2 Hidden Layers 3 Hidden Layers 100 Hidden Units 100 Hidden Units 100 Hidden Units 92.48% 95.09% 95.54% Model A LSTM Model B LSTM Model C LSTM 1 Hidden Layer 2 Hidden Layers 3 Hidden Layers 100 Hidden Units 100 Hidden Units 100 Hidden Units 96.05% 95.24% 91.22%"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#deep-learning-notes","title":"Deep Learning Notes","text":"<ul> <li>2 ways to expand a recurrent neural network<ul> <li>More hidden units<ul> <li><code>(o, i, f, g) gates</code></li> </ul> </li> <li>More hidden layers</li> </ul> </li> <li>Cons<ul> <li>Need a larger dataset<ul> <li>Curse of dimensionality</li> </ul> </li> <li>Does not necessarily mean higher accuracy</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#3-building-a-recurrent-neural-network-with-pytorch-gpu","title":"3. Building a Recurrent Neural Network with PyTorch (GPU)","text":""},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#model-a-3-hidden-layers","title":"Model A: 3 Hidden Layers","text":"<p>GPU: 2 things must be on GPU - <code>model</code> - <code>tensors</code></p>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#steps_3","title":"Steps","text":"<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> </ul> <p>3 Hidden Layer LSTM on GPU</p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n'''\nSTEP 1: LOADING DATASET\n'''\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\n\nclass LSTMModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):\n        super(LSTMModel, self).__init__()\n        # Hidden dimensions\n        self.hidden_dim = hidden_dim\n\n        # Number of hidden layers\n        self.layer_dim = layer_dim\n\n        # Building your LSTM\n        # batch_first=True causes input/output tensors to be of shape\n        # (batch_dim, seq_dim, feature_dim)\n        self.lstm = nn.LSTM(input_dim, hidden_dim, layer_dim, batch_first=True)\n\n        # Readout layer\n        self.fc = nn.Linear(hidden_dim, output_dim)\n\n    def forward(self, x):\n        # Initialize hidden state with zeros\n        #######################\n        #  USE GPU FOR MODEL  #\n        #######################\n        h0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).requires_grad_().to(device)\n\n        # Initialize cell state\n        c0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).requires_grad_().to(device)\n\n        # One time step\n        out, (hn, cn) = self.lstm(x, (h0.detach(), c0.detach()))\n\n        # Index hidden state of last time step\n        # out.size() --&gt; 100, 28, 100\n        # out[:, -1, :] --&gt; 100, 100 --&gt; just want last time step hidden states! \n        out = self.fc(out[:, -1, :]) \n        # out.size() --&gt; 100, 10\n        return out\n\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28\nhidden_dim = 100\nlayer_dim = 3  # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER\noutput_dim = 10\n\nmodel = LSTMModel(input_dim, hidden_dim, layer_dim, output_dim)\n\n#######################\n#  USE GPU FOR MODEL  #\n#######################\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)  \n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\n\n# Number of steps to unroll\nseq_dim = 28  \n\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as Variable\n        #######################\n        #  USE GPU FOR MODEL  #\n        #######################\n        images = images.view(-1, seq_dim, input_dim).requires_grad_().to(device)\n        labels = labels.to(device)\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        # outputs.size() --&gt; 100, 10\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                #######################\n                #  USE GPU FOR MODEL  #\n                #######################\n                images = images.view(-1, seq_dim, input_dim).to(device)\n                labels = labels.to(device)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                #######################\n                #  USE GPU FOR MODEL  #\n                #######################\n                if torch.cuda.is_available():\n                    correct += (predicted.cpu() == labels.cpu()).sum()\n                else:\n                    correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 2.3068575859069824. Accuracy: 11\nIteration: 1000. Loss: 2.291989803314209. Accuracy: 14\nIteration: 1500. Loss: 1.909593105316162. Accuracy: 28\nIteration: 2000. Loss: 0.7345633506774902. Accuracy: 71\nIteration: 2500. Loss: 0.45030108094215393. Accuracy: 86\nIteration: 3000. Loss: 0.2627193331718445. Accuracy: 89\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#summary","title":"Summary","text":"<p>We've learnt to...</p> <p>Success</p> <ul> <li> RNN transition to LSTM</li> <li> LSTM Models in PyTorch<ul> <li> Model A: 1 Hidden Layer LSTM</li> <li> Model B: 2 Hidden Layer LSTM</li> <li> Model C: 3 Hidden Layer LSTM</li> </ul> </li> <li> Models Variation in Code<ul> <li> Modifying only step 4</li> </ul> </li> <li> Ways to Expand Model\u2019s Capacity<ul> <li> More hidden units</li> <li> More hidden layers</li> </ul> </li> <li> Cons of Expanding Capacity<ul> <li> Need more data</li> <li> Does not necessarily mean higher accuracy</li> </ul> </li> <li> GPU Code<ul> <li> 2 things on GPU<ul> <li> model</li> <li> tensors</li> </ul> </li> <li> Modifying only Step 3, 4 and 7</li> </ul> </li> <li> 7 Step Model Building Recap<ul> <li> Step 1: Load Dataset</li> <li> Step 2: Make Dataset Iterable</li> <li> Step 3: Create Model Class</li> <li> Step 4: Instantiate Model Class</li> <li> Step 5: Instantiate Loss Class</li> <li> Step 6: Instantiate Optimizer Class</li> <li> Step 7: Train Model</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/#citation","title":"Citation","text":"<p>If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.</p> <p></p>"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/","title":"Matrices with PyTorch","text":"<p>Run Jupyter Notebook</p> <p>You can run the code for this section in this jupyter notebook link.</p>"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#matrices","title":"Matrices","text":""},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#matrices-brief-introduction","title":"Matrices Brief Introduction","text":"<ul> <li> Basic definition: rectangular array of numbers.</li> <li> Tensors (PyTorch)</li> <li> Ndarrays (NumPy)</li> </ul> <p>2 x 2 Matrix (R x C)</p> 1 1 1 1 <p>2 x 3 Matrix</p> 1 1 1 1 1 1"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#creating-matrices","title":"Creating Matrices","text":"<p>Create list</p> <pre><code># Creating a 2x2 array\narr = [[1, 2], [3, 4]]\nprint(arr)\n</code></pre> <pre><code>[[1, 2], [3, 4]]\n</code></pre> <p>Create numpy array via list</p> <p><pre><code>import numpy as np\n</code></pre> <pre><code># Convert to NumPy\nnp.array(arr)\n</code></pre></p> <pre><code>array([[1, 2],\n       [3, 4]])\n</code></pre> <p>Convert numpy array to PyTorch tensor</p> <pre><code>import torch\n</code></pre> <pre><code># Convert to PyTorch Tensor\ntorch.Tensor(arr)\n</code></pre> <pre><code>1  2\n3  4\n[torch.FloatTensor of size 2x2]\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#create-matrices-with-default-values","title":"Create Matrices with Default Values","text":"<p>Create 2x2 numpy array of 1's</p> <pre><code>np.ones((2, 2))\n</code></pre> <pre><code>array([[ 1.,  1.],\n       [ 1.,  1.]])\n</code></pre> <p>Create 2x2 torch tensor of 1's</p> <pre><code>torch.ones((2, 2))\n</code></pre> <pre><code> 1  1\n 1  1\n[torch.FloatTensor of size 2x2]\n</code></pre> <p>Create 2x2 numpy array of random numbers</p> <pre><code>np.random.rand(2, 2)\n</code></pre> <pre><code>array([[ 0.68270631,  0.87721678],\n       [ 0.07420986,  0.79669375]])\n</code></pre> <p>Create 2x2 PyTorch tensor of random numbers</p> <pre><code>torch.rand(2, 2)\n</code></pre> <pre><code>0.3900  0.8268\n0.3888  0.5914\n[torch.FloatTensor of size 2x2]\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#seeds-for-reproducibility","title":"Seeds for Reproducibility","text":"<p>Why do we need seeds?</p> <p>We need seeds to enable reproduction of experimental results. This becomes critical later on where you can easily let people reproduce your code's output exactly as you've produced.</p> <p>Create seed to enable fixed numbers for random number generation </p> <pre><code># Seed\nnp.random.seed(0)\nnp.random.rand(2, 2)\n</code></pre> <pre><code>array([[ 0.5488135 ,  0.71518937],\n       [ 0.60276338,  0.54488318]])\n</code></pre> <p>Repeat random array generation to check</p> <p>If you do not set the seed, you would not get the same set of numbers like here. <pre><code># Seed\nnp.random.seed(0)\nnp.random.rand(2, 2)\n</code></pre></p> <pre><code>array([[ 0.5488135 ,  0.71518937],\n       [ 0.60276338,  0.54488318]])\n</code></pre> <p>Create a numpy array without seed</p> <p>Notice how you get different numbers compared to the first 2 tries? <pre><code># No seed\nnp.random.rand(2, 2)\n</code></pre></p> <pre><code>array([[ 0.56804456,  0.92559664],\n       [ 0.07103606,  0.0871293 ]])\n</code></pre> <p>Repeat numpy array generation without seed</p> <p>You get the point now, you get a totally different set of numbers. <pre><code># No seed\nnp.random.rand(2, 2)\n</code></pre></p> <pre><code>array([[ 0.0202184 ,  0.83261985],\n       [ 0.77815675,  0.87001215]])\n</code></pre> <p>Create a PyTorch tensor with a fixed seed</p> <pre><code># Torch Seed\ntorch.manual_seed(0)\ntorch.rand(2, 2)\n</code></pre> <pre><code>0.5488  0.5928\n0.7152  0.8443\n[torch.FloatTensor of size 2x2]\n</code></pre> <p>Repeat creating a PyTorch fixed seed tensor</p> <pre><code># Torch Seed\ntorch.manual_seed(0)\ntorch.rand(2, 2)\n</code></pre> <pre><code>0.5488  0.5928\n0.7152  0.8443\n[torch.FloatTensor of size 2x2]\n</code></pre> <p>Creating a PyTorch tensor without seed</p> <p>Like with a numpy array of random numbers without seed, you will not get the same results as above. <pre><code># Torch No Seed\ntorch.rand(2, 2)\n</code></pre></p> <pre><code>0.6028  0.8579\n0.5449  0.8473\n[torch.FloatTensor of size 2x2]\n</code></pre> <p>Repeat creating a PyTorch tensor without seed</p> <p>Notice how these are different numbers again? <pre><code># Torch No Seed\ntorch.rand(2, 2)\n</code></pre></p> <pre><code>0.4237  0.6236\n0.6459  0.3844\n[torch.FloatTensor of size 2x2]\n</code></pre> <p>Seed for GPU is different for now...</p> <p>Fix a seed for GPU tensors</p> <p>When you conduct deep learning experiments, typically you want to use GPUs to accelerate your computations and fixing seed for tensors on GPUs is different from CPUs as we have done above.  <pre><code>if torch.cuda.is_available():\n    torch.cuda.manual_seed_all(0)\n</code></pre></p>"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#numpy-and-torch-bridge","title":"NumPy and Torch Bridge","text":""},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#numpy-to-torch","title":"NumPy to Torch","text":"<p>Create a numpy array of 1's</p> <pre><code># Numpy array\nnp_array = np.ones((2, 2))\n</code></pre> <pre><code>print(np_array)\n</code></pre> <pre><code>[[ 1.  1.]\n[ 1.  1.]]\n</code></pre> <p>Get the type of class for the numpy array</p> <pre><code>print(type(np_array))\n</code></pre> <pre><code>&lt;class 'numpy.ndarray'&gt;\n</code></pre> <p>Convert numpy array to PyTorch tensor</p> <pre><code># Convert to Torch Tensor\ntorch_tensor = torch.from_numpy(np_array)\n</code></pre> <pre><code>print(torch_tensor)\n</code></pre> <pre><code> 1  1\n 1  1\n[torch.DoubleTensor of size 2x2]\n</code></pre> <p>Get type of class for PyTorch tensor</p> <p>Notice how it shows it's a torch DoubleTensor? There're actually tensor types and it depends on the numpy data type. <pre><code>print(type(torch_tensor))\n</code></pre></p> <pre><code>&lt;class 'torch.DoubleTensor'&gt;\n</code></pre> <p>Create PyTorch tensor from a different numpy datatype</p> <p>You will get an error running this code because PyTorch tensor don't support all datatype.  <pre><code># Data types matter: intentional error\nnp_array_new = np.ones((2, 2), dtype=np.int8)\ntorch.from_numpy(np_array_new)\n</code></pre></p> <pre><code>---------------------------------------------------------------------------\nRuntimeError                              Traceback (most recent call last)\n\n&lt;ipython-input-57-b8b085f9b39d&gt; in &lt;module&gt;()\n      1 # Data types matter\n      2 np_array_new = np.ones((2, 2), dtype=np.int8)\n----&gt; 3 torch.from_numpy(np_array_new)\n\n\nRuntimeError: can't convert a given np.ndarray to a tensor - it has an invalid type. The only supported types are: double, float, int64, int32, and uint8.\n</code></pre> <p>What conversion support does Numpy to PyTorch tensor bridge gives?</p> <ul> <li><code>double</code></li> <li><code>float</code> </li> <li><code>int64</code>, <code>int32</code>, <code>uint8</code> </li> </ul> <p>Create PyTorch long tensor</p> <p>See how a int64 numpy array gives you a PyTorch long tensor? <pre><code># Data types matter\nnp_array_new = np.ones((2, 2), dtype=np.int64)\ntorch.from_numpy(np_array_new)\n</code></pre></p> <pre><code>1  1\n1  1\n[torch.LongTensor of size 2x2]\n</code></pre> <p>Create PyTorch int tensor</p> <pre><code># Data types matter\nnp_array_new = np.ones((2, 2), dtype=np.int32)\ntorch.from_numpy(np_array_new)\n</code></pre> <pre><code>1  1\n1  1\n[torch.IntTensor of size 2x2]\n</code></pre> <p>Create PyTorch byte tensor</p> <pre><code># Data types matter\nnp_array_new = np.ones((2, 2), dtype=np.uint8)\ntorch.from_numpy(np_array_new)\n</code></pre> <pre><code>1  1\n1  1\n[torch.ByteTensor of size 2x2]\n</code></pre> <p>Create PyTorch Double Tensor</p> <pre><code># Data types matter\nnp_array_new = np.ones((2, 2), dtype=np.float64)\ntorch.from_numpy(np_array_new)\n</code></pre> <p>Alternatively you can do this too via <code>np.double</code></p> <pre><code># Data types matter\nnp_array_new = np.ones((2, 2), dtype=np.double)\ntorch.from_numpy(np_array_new)\n</code></pre> <pre><code>1  1\n1  1\n[torch.DoubleTensor of size 2x2]\n</code></pre> <p>Create PyTorch Float Tensor</p> <pre><code># Data types matter\nnp_array_new = np.ones((2, 2), dtype=np.float32)\ntorch.from_numpy(np_array_new)\n</code></pre> <pre><code>1  1\n1  1\n[torch.FloatTensor of size 2x2]\n</code></pre> <p>Summary</p> <p>Tensor Type Bug Guide</p> <p>These things don't matter much now. But later when you see error messages that require these particular tensor types, refer to this guide!</p> NumPy Array Type Torch Tensor Type int64 LongTensor int32 IntegerTensor uint8 ByteTensor float64 DoubleTensor float32 FloatTensor double DoubleTensor"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#torch-to-numpy","title":"Torch to NumPy","text":"<p>Create PyTorch tensor of 1's</p> <p>You would realize this defaults to a float tensor by default if you do this.</p> <pre><code>torch_tensor = torch.ones(2, 2)\n</code></pre> <pre><code>type(torch_tensor)\n</code></pre> <pre><code>torch.FloatTensor\n</code></pre> <p>Convert tensor to numpy</p> <p>It's as simple as this.</p> <pre><code>torch_to_numpy = torch_tensor.numpy()\n</code></pre> <pre><code>type(torch_to_numpy)\n</code></pre> <pre><code># Wowza, we did it.\nnumpy.ndarray\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#tensors-on-cpu-vs-gpu","title":"Tensors on CPU vs GPU","text":"<p>Move tensor to CPU and back</p> <p>This by default creates a tensor on CPU. You do not need to do anything. <pre><code># CPU\ntensor_cpu = torch.ones(2, 2)\n</code></pre></p> <p>If you would like to send a tensor to your GPU, you just need to do a simple <code>.cuda()</code></p> <pre><code># CPU to GPU\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\ntensor_cpu.to(device)\n</code></pre> <p>And if you want to move that tensor on the GPU back to the CPU, just do the following.</p> <pre><code># GPU to CPU\ntensor_cpu.cpu()\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#tensor-operations","title":"Tensor Operations","text":""},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#resizing-tensor","title":"Resizing Tensor","text":"<p>Creating a 2x2 tensor</p> <pre><code>a = torch.ones(2, 2)\nprint(a)\n</code></pre> <pre><code>1  1\n1  1\n[torch.FloatTensor of size 2x2]\n</code></pre> <p>Getting size of tensor</p> <pre><code>print(a.size())\n</code></pre> <pre><code>torch.Size([2, 2])\n</code></pre> <p>Resize tensor to 4x1</p> <pre><code>a.view(4)\n</code></pre> <pre><code>1\n1\n1\n1\n[torch.FloatTensor of size 4]\n</code></pre> <p>Get size of resized tensor</p> <pre><code>a.view(4).size()\n</code></pre> <pre><code>torch.Size([4])\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#element-wise-addition","title":"Element-wise Addition","text":"<p>Creating first 2x2 tensor</p> <pre><code>a = torch.ones(2, 2)\nprint(a)\n</code></pre> <pre><code>1  1\n1  1\n[torch.FloatTensor of size 2x2]\n</code></pre> <p>Creating second 2x2 tensor</p> <pre><code>b = torch.ones(2, 2)\nprint(b)\n</code></pre> <pre><code>1  1\n1  1\n[torch.FloatTensor of size 2x2]\n</code></pre> <p>Element-wise addition of 2 tensors</p> <pre><code># Element-wise addition\nc = a + b\nprint(c)\n</code></pre> <pre><code> 2  2\n 2  2\n[torch.FloatTensor of size 2x2]\n</code></pre> <p>Alternative element-wise addition of 2 tensors</p> <pre><code># Element-wise addition\nc = torch.add(a, b)\nprint(c)\n</code></pre> <pre><code> 2  2\n 2  2\n[torch.FloatTensor of size 2x2]\n</code></pre> <p>In-place element-wise addition</p> <p>This would replace the c tensor values with the new addition. </p> <pre><code># In-place addition\nprint('Old c tensor')\nprint(c)\n\nc.add_(a)\n\nprint('-'*60)\nprint('New c tensor')\nprint(c)\n</code></pre> <pre><code>Old c tensor\n\n 2  2\n 2  2\n[torch.FloatTensor of size 2x2]\n\n------------------------------------------------------------\nNew c tensor\n\n 3  3\n 3  3\n[torch.FloatTensor of size 2x2]\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#element-wise-subtraction","title":"Element-wise Subtraction","text":"<p>Check values of tensor a and b'</p> <p>Take note that you've created tensor a and b of sizes 2x2 filled with 1's each above.  <pre><code>print(a)\nprint(b)\n</code></pre></p> <pre><code> 1  1\n 1  1\n[torch.FloatTensor of size 2x2]\n\n\n 1  1\n 1  1\n[torch.FloatTensor of size 2x2]\n</code></pre> <p>Element-wise subtraction: method 1</p> <pre><code>a - b\n</code></pre> <pre><code>0  0\n0  0\n[torch.FloatTensor of size 2x2]\n</code></pre> <p>Element-wise subtraction: method 2</p> <pre><code># Not in-place\nprint(a.sub(b))\nprint(a)\n</code></pre> <pre><code>0  0\n0  0\n[torch.FloatTensor of size 2x2]\n\n\n1  1\n1  1\n[torch.FloatTensor of size 2x2]\n</code></pre> <p>Element-wise subtraction: method 3</p> <p>This will replace a with the final result filled with 2's <pre><code># Inplace\nprint(a.sub_(b))\nprint(a)\n</code></pre></p> <pre><code>0  0\n0  0\n[torch.FloatTensor of size 2x2]\n\n\n0  0\n0  0\n[torch.FloatTensor of size 2x2]\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#element-wise-multiplication","title":"Element-Wise Multiplication","text":"<p>Create tensor a and b of sizes 2x2 filled with 1's and 0's</p> <pre><code>a = torch.ones(2, 2)\nprint(a)\nb = torch.zeros(2, 2)\nprint(b)\n</code></pre> <pre><code>1  1\n1  1\n[torch.FloatTensor of size 2x2]\n\n\n0  0\n0  0\n[torch.FloatTensor of size 2x2]\n</code></pre> <p>Element-wise multiplication: method 1</p> <pre><code>a * b\n</code></pre> <pre><code>0  0\n0  0\n[torch.FloatTensor of size 2x2]\n</code></pre> <p>Element-wise multiplication: method 2</p> <pre><code># Not in-place\nprint(torch.mul(a, b))\nprint(a)\n</code></pre> <pre><code>0  0\n0  0\n[torch.FloatTensor of size 2x2]\n\n1  1\n1  1\n[torch.FloatTensor of size 2x2]\n</code></pre> <p>Element-wise multiplication: method 3</p> <pre><code># In-place\nprint(a.mul_(b))\nprint(a)\n</code></pre> <pre><code>0  0\n0  0\n[torch.FloatTensor of size 2x2]\n\n0  0\n0  0\n[torch.FloatTensor of size 2x2]\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#element-wise-division","title":"Element-Wise Division","text":"<p>Create tensor a and b of sizes 2x2 filled with 1's and 0's</p> <pre><code>a = torch.ones(2, 2)\nprint(a)\nb = torch.zeros(2, 2)\nprint(b)\n</code></pre> <pre><code>1  1\n1  1\n[torch.FloatTensor of size 2x2]\n\n\n0  0\n0  0\n[torch.FloatTensor of size 2x2]\n</code></pre> <p>Element-wise division: method 1</p> <pre><code>b / a\n</code></pre> <pre><code>0  0\n0  0\n[torch.FloatTensor of size 2x2]\n</code></pre> <p>Element-wise division: method 2</p> <pre><code>torch.div(b, a)\n</code></pre> <pre><code>0  0\n0  0\n[torch.FloatTensor of size 2x2]\n</code></pre> <p>Element-wise division: method 3</p> <pre><code># Inplace\nb.div_(a)\n</code></pre> <pre><code>0  0\n0  0\n[torch.FloatTensor of size 2x2]\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#tensor-mean","title":"Tensor Mean","text":"\\[1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 = 55\\] \\[ mean = 55 /10 = 5.5 \\] <p>Create tensor of size 10 filled from 1 to 10</p> <pre><code>a = torch.Tensor([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])\na.size()\n</code></pre> <pre><code>torch.Size([10])\n</code></pre> <p>Get tensor mean</p> <p>Here we get 5.5 as we've calculated manually above.</p> <pre><code>a.mean(dim=0)\n</code></pre> <pre><code>5.5000\n[torch.FloatTensor of size 1]\n</code></pre> <p>Get tensor mean on second dimension</p> <p>Here we get an error because the tensor is of size 10 and not 10x1 so there's no second dimension to calculate.</p> <pre><code>a.mean(dim=1)\n</code></pre> <pre><code>RuntimeError                              Traceback (most recent call last)\n\n&lt;ipython-input-7-81aec0cf1c00&gt; in &lt;module&gt;()\n----&gt; 1 a.mean(dim=1)\n\n\nRuntimeError: dimension out of range (expected to be in range of [-1, 0], but got 1)\n</code></pre> <p>Create a 2x10 Tensor, of 1-10 digits each</p> <p><pre><code>a = torch.Tensor([[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]])\n</code></pre> <pre><code>a.size()\n</code></pre></p> <pre><code>torch.Size([2, 10])\n</code></pre> <p>Get tensor mean on second dimension</p> <p>Here we won't get an error like previously because we've a tensor of size 2x10</p> <pre><code>a.mean(dim=1)\n</code></pre> <pre><code> 5.5000\n 5.5000\n[torch.FloatTensor of size 2x1]\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#tensor-standard-deviation","title":"Tensor Standard Deviation","text":"<p>Get standard deviation of tensor</p> <pre><code>a = torch.Tensor([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])\na.std(dim=0)\n</code></pre> <pre><code> 3.0277\n[torch.FloatTensor of size 1]\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#summary","title":"Summary","text":"<p>We've learnt to...</p> <p>Success</p> <ul> <li> Create Matrices</li> <li> Create Matrices with Default Initialization Values<ul> <li> Zeros </li> <li> Ones</li> </ul> </li> <li> Initialize Seeds for Reproducibility on GPU and CPU</li> <li> Convert Matrices: NumPy to Torch and Torch to NumPy</li> <li> Move Tensors: CPU to GPU and GPU to CPU</li> <li> Run Important Tensor Operations<ul> <li> Element-wise addition, subtraction, multiplication and division</li> <li> Resize</li> <li> Calculate mean </li> <li> Calculate standard deviation</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_matrices/#citation","title":"Citation","text":"<p>If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.</p> <p></p>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/","title":"Recurrent Neural Network with PyTorch","text":"<p>Run Jupyter Notebook</p> <p>You can run the code for this section in this jupyter notebook link.</p>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#about-recurrent-neural-network","title":"About Recurrent Neural Network","text":""},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#feedforward-neural-networks-transition-to-1-layer-recurrent-neural-networks-rnn","title":"Feedforward Neural Networks Transition to 1 Layer Recurrent Neural Networks (RNN)","text":"<ul> <li>RNN is essentially an FNN but with a hidden layer (non-linear output) that passes on information to the next FNN</li> <li>Compared to an FNN, we've one additional set of weight and bias that allows information to flow from one FNN to another FNN sequentially that allows time-dependency.</li> <li>The diagram below shows the only difference between an FNN and a RNN. </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#2-layer-rnn-breakdown","title":"2 Layer RNN Breakdown","text":""},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#building-a-recurrent-neural-network-with-pytorch","title":"Building a Recurrent Neural Network with PyTorch","text":""},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#model-a-1-hidden-layer-relu","title":"Model A: 1 Hidden Layer (ReLU)","text":"<ul> <li>Unroll 28 time steps<ul> <li>Each step input size: 28 x 1</li> <li>Total per unroll: 28 x 28<ul> <li>Feedforward Neural Network input size: 28 x 28 </li> </ul> </li> </ul> </li> <li>1 Hidden layer</li> <li>ReLU Activation Function</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#steps","title":"Steps","text":"<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#step-1-loading-mnist-train-dataset","title":"Step 1: Loading MNIST Train Dataset","text":"<p>Images from 1 to 9</p> <p>Looking into the MNIST Dataset</p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n</code></pre> <pre><code>train_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n</code></pre> <p>We would have 60k training images of size 28 x 28 pixels.</p> <pre><code>print(train_dataset.train_data.size())\n</code></pre> <pre><code>print(train_dataset.train_labels.size())\n</code></pre> <p>Here we would have 10k testing images of the same size, 28 x 28 pixels.</p> <pre><code>print(test_dataset.test_data.size())\n</code></pre> <pre><code>print(test_dataset.test_labels.size())\n</code></pre> <pre><code>torch.Size([60000, 28, 28])\n\ntorch.Size([60000])\n\ntorch.Size([10000, 28, 28])\n\ntorch.Size([10000])\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#step-2-make-dataset-iterable","title":"Step 2: Make Dataset Iterable","text":"<p>Creating iterable objects to loop through subsequently</p> <pre><code>batch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#step-3-create-model-class","title":"Step 3: Create Model Class","text":"<p>1 Layer RNN</p> <p></p> <pre><code>class RNNModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):\n        super(RNNModel, self).__init__()\n        # Hidden dimensions\n        self.hidden_dim = hidden_dim\n\n        # Number of hidden layers\n        self.layer_dim = layer_dim\n\n        # Building your RNN\n        # batch_first=True causes input/output tensors to be of shape\n        # (batch_dim, seq_dim, input_dim)\n        # batch_dim = number of samples per batch\n        self.rnn = nn.RNN(input_dim, hidden_dim, layer_dim, batch_first=True, nonlinearity='relu')\n\n        # Readout layer\n        self.fc = nn.Linear(hidden_dim, output_dim)\n\n    def forward(self, x):\n        # Initialize hidden state with zeros\n        # (layer_dim, batch_size, hidden_dim)\n        h0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).requires_grad_()\n\n        # We need to detach the hidden state to prevent exploding/vanishing gradients\n        # This is part of truncated backpropagation through time (BPTT)\n        out, hn = self.rnn(x, h0.detach())\n\n        # Index hidden state of last time step\n        # out.size() --&gt; 100, 28, 10\n        # out[:, -1, :] --&gt; 100, 10 --&gt; just want last time step hidden states! \n        out = self.fc(out[:, -1, :]) \n        # out.size() --&gt; 100, 10\n        return out\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#step-4-instantiate-model-class","title":"Step 4: Instantiate Model Class","text":"<ul> <li>28 time steps<ul> <li>Each time step: input dimension = 28</li> </ul> </li> <li>1 hidden layer</li> <li>MNIST 1-9 digits \\(\\rightarrow\\) output dimension = 10</li> </ul> <p>Instantiate model class and assign to an object</p> <pre><code>input_dim = 28\nhidden_dim = 100\nlayer_dim = 1\noutput_dim = 10\n</code></pre> <pre><code>model = RNNModel(input_dim, hidden_dim, layer_dim, output_dim)\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#step-5-instantiate-loss-class","title":"Step 5: Instantiate Loss Class","text":"<ul> <li>Recurrent Neural Network: Cross Entropy Loss<ul> <li>Convolutional Neural Network: Cross Entropy Loss</li> <li>Feedforward Neural Network: Cross Entropy Loss</li> <li>Logistic Regression: Cross Entropy Loss</li> <li>Linear Regression: MSE</li> </ul> </li> </ul> <p>Cross Entropy Loss for Classification Task</p> <pre><code>criterion = nn.CrossEntropyLoss()\n</code></pre> <p>Cross Entropy vs MSE</p> <p>Take note that there are cases where RNN, CNN and FNN use MSE as a loss function.</p> <p>We use cross entropy for classification tasks (predicting 0-9 digits in MNIST for example).</p> <p>And we use MSE for regression tasks (predicting temperatures in every December in San Francisco for example).</p>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#step-6-instantiate-optimizer-class","title":"Step 6: Instantiate Optimizer Class","text":"<ul> <li>Simplified equation<ul> <li>\\(\\theta = \\theta - \\eta \\cdot \\nabla_\\theta\\)<ul> <li>\\(\\theta\\): parameters (our tensors with gradient accumulation abilities)</li> <li>\\(\\eta\\): learning rate (how fast we want to learn)</li> <li>\\(\\nabla_\\theta\\): gradients of loss with respect to the model's parameters</li> </ul> </li> </ul> </li> <li>Even simplier equation<ul> <li><code>parameters = parameters - learning_rate * parameters_gradients</code></li> <li>At every iteration, we update our model's parameters</li> </ul> </li> </ul> <pre><code>learning_rate = 0.01\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)  \n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#parameters-in-depth","title":"Parameters In-Depth","text":"<ul> <li>Input to Hidden Layer Affine Function<ul> <li>A1, B1</li> </ul> </li> <li>Hidden Layer to Output Affine Function<ul> <li>A2, B2</li> </ul> </li> <li>Hidden Layer to Hidden Layer Affine Function<ul> <li>A3, B3</li> </ul> </li> </ul> <p>Total groups of parameters</p> <p>We should have 6 groups as shown above.</p> <pre><code>len(list(model.parameters()))\n</code></pre> <pre><code>6\n</code></pre> <p>Input to Hidden Weight</p> <p>Remember we defined our hidden layer to have a size of 100. Because our input is a size of 28 at each time step, this gives rise to a weight matrix of 100 x 28.</p> <pre><code># Input --&gt; Hidden (A1)\nlist(model.parameters())[0].size()\n</code></pre> <pre><code>torch.Size([100, 28])\n</code></pre> <p>Input to Hidden Bias</p> <pre><code># Input --&gt; Hidden BIAS (B1)\nlist(model.parameters())[2].size()\n</code></pre> <pre><code>torch.Size([100])\n</code></pre> <p>Hidden to Hidden</p> <pre><code># Hidden --&gt; Hidden (A3)\nlist(model.parameters())[1].size()\n</code></pre> <pre><code>torch.Size([100, 100])\n</code></pre> <p>Hidden to Hidden Bias</p> <pre><code># Hidden --&gt; Hidden BIAS(B3)\nlist(model.parameters())[3].size()\n</code></pre> <pre><code>torch.Size([100])\n</code></pre> <p>Hidden to Output</p> <pre><code># Hidden --&gt; Output (A2)\nlist(model.parameters())[4].size()\n</code></pre> <pre><code>torch.Size([10, 100])\n</code></pre> <p>Hidden to Output Bias</p> <pre><code># Hidden --&gt; Output BIAS (B2)\nlist(model.parameters())[5].size()\n</code></pre> <pre><code>torch.Size([10])\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#step-7-train-model","title":"Step 7: Train Model","text":"<ul> <li>Process <ol> <li>Convert inputs/labels to tensors with gradient accumulation abilities<ul> <li>RNN Input: (1, 28)</li> <li>CNN Input: (1, 28, 28) </li> <li>FNN Input: (1, 28*28)</li> </ul> </li> <li>Clear gradient buffets</li> <li>Get output given inputs </li> <li>Get loss</li> <li>Get gradients w.r.t. parameters</li> <li>Update parameters using gradients<ul> <li><code>parameters = parameters - learning_rate * parameters_gradients</code></li> </ul> </li> <li>REPEAT</li> </ol> </li> </ul> <p>Same 7 step process for training models</p> <pre><code># Number of steps to unroll\nseq_dim = 28  \n\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        model.train()\n        # Load images as tensors with gradient accumulation abilities\n        images = images.view(-1, seq_dim, input_dim).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        # outputs.size() --&gt; 100, 10\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            model.eval()\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Load images to a Torch tensors with gradient accumulation abilities\n                images = images.view(-1, seq_dim, input_dim)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 2.301494836807251. Accuracy: 12\nIteration: 1000. Loss: 2.2986037731170654. Accuracy: 14\nIteration: 1500. Loss: 2.278566598892212. Accuracy: 18\nIteration: 2000. Loss: 2.169614315032959. Accuracy: 21\nIteration: 2500. Loss: 1.1662731170654297. Accuracy: 51\nIteration: 3000. Loss: 0.9290509223937988. Accuracy: 71\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#model-b-2-hidden-layer-relu","title":"Model B: 2 Hidden Layer (ReLU)","text":"<ul> <li>Unroll 28 time steps<ul> <li>Each step input size: 28 x 1</li> <li>Total per unroll: 28 x 28<ul> <li>Feedforward Neural Network inpt size: 28 x 28 </li> </ul> </li> </ul> </li> <li>2 Hidden layer</li> <li>ReLU Activation Function</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#steps_1","title":"Steps","text":"<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> </ul> <p>2 Hidden Layer + ReLU</p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n'''\nSTEP 1: LOADING DATASET\n'''\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\n\nclass RNNModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):\n        super(RNNModel, self).__init__()\n        # Hidden dimensions\n        self.hidden_dim = hidden_dim\n\n        # Number of hidden layers\n        self.layer_dim = layer_dim\n\n        # Building your RNN\n        # batch_first=True causes input/output tensors to be of shape\n        # (batch_dim, seq_dim, feature_dim)\n        self.rnn = nn.RNN(input_dim, hidden_dim, layer_dim, batch_first=True, nonlinearity='relu')\n\n        # Readout layer\n        self.fc = nn.Linear(hidden_dim, output_dim)\n\n    def forward(self, x):\n        # Initialize hidden state with zeros\n        h0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).requires_grad_()\n\n        # We need to detach the hidden state to prevent exploding/vanishing gradients\n        # This is part of truncated backpropagation through time (BPTT)\n        out, hn = self.rnn(x, h0.detach())\n\n        # Index hidden state of last time step\n        # out.size() --&gt; 100, 28, 100\n        # out[:, -1, :] --&gt; 100, 100 --&gt; just want last time step hidden states! \n        out = self.fc(out[:, -1, :]) \n        # out.size() --&gt; 100, 10\n        return out\n\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28\nhidden_dim = 100\nlayer_dim = 2  # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER\noutput_dim = 10\n\nmodel = RNNModel(input_dim, hidden_dim, layer_dim, output_dim)\n\n# JUST PRINTING MODEL &amp; PARAMETERS \nprint(model)\nprint(len(list(model.parameters())))\nfor i in range(len(list(model.parameters()))):\n    print(list(model.parameters())[i].size())\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.01\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)  \n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\n\n# Number of steps to unroll\nseq_dim = 28  \n\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        model.train()\n        # Load images as tensors with gradient accumulation abilities\n        images = images.view(-1, seq_dim, input_dim).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        # outputs.size() --&gt; 100, 10\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            model.eval()\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Resize images\n                images = images.view(-1, seq_dim, input_dim)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>RNNModel(\n  (rnn): RNN(28, 100, num_layers=2, batch_first=True)\n  (fc): Linear(in_features=100, out_features=10, bias=True)\n)\n\n10\n\ntorch.Size([100, 28])\ntorch.Size([100, 100])\ntorch.Size([100])\ntorch.Size([100])\ntorch.Size([100, 100])\ntorch.Size([100, 100])\ntorch.Size([100])\ntorch.Size([100])\ntorch.Size([10, 100])\ntorch.Size([10])\n\nIteration: 500. Loss: 2.3019518852233887. Accuracy: 11\nIteration: 1000. Loss: 2.299217700958252. Accuracy: 11\nIteration: 1500. Loss: 2.279090166091919. Accuracy: 14\nIteration: 2000. Loss: 2.126953125. Accuracy: 25\nIteration: 2500. Loss: 1.356347680091858. Accuracy: 57\nIteration: 3000. Loss: 0.7377720475196838. Accuracy: 69\n</code></pre> <ul> <li>10 sets of parameters</li> <li>First hidden Layer<ul> <li>\\(A_1 = [100, 28]\\)</li> <li>\\(A_3 = [100, 100]\\)</li> <li>\\(B_1 = [100]\\)</li> <li>\\(B_3 = [100]\\)</li> </ul> </li> <li>Second hidden layer<ul> <li>\\(A_2 = [100, 100]\\)</li> <li>\\(A_5 = [100, 100]\\)</li> <li>\\(B_2 = [100]\\)</li> <li>\\(B_5 = [100]\\)</li> </ul> </li> <li>Readout layer<ul> <li>\\(A_4 = [10, 100]\\)</li> <li>\\(B_4 = [10]\\)</li> </ul> </li> </ul> <p></p>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#model-c-2-hidden-layer","title":"Model C: 2 Hidden Layer","text":"<ul> <li>Unroll 28 time steps<ul> <li>Each step input size: 28 x 1</li> <li>Total per unroll: 28 x 28<ul> <li>Feedforward Neural Network inpt size: 28 x 28 </li> </ul> </li> </ul> </li> <li>2 Hidden layer</li> <li>Tanh Activation Function</li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#steps_2","title":"Steps","text":"<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> </ul> <p>!!! \"2 Hidden + ReLU\"     <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n'''\nSTEP 1: LOADING DATASET\n'''\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\n\nclass RNNModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):\n        super(RNNModel, self).__init__()\n        # Hidden dimensions\n        self.hidden_dim = hidden_dim\n\n        # Number of hidden layers\n        self.layer_dim = layer_dim\n\n        # Building your RNN\n        # batch_first=True causes input/output tensors to be of shape\n        # (batch_dim, seq_dim, feature_dim)\n        self.rnn = nn.RNN(input_dim, hidden_dim, layer_dim, batch_first=True, nonlinearity='tanh')\n\n        # Readout layer\n        self.fc = nn.Linear(hidden_dim, output_dim)\n\n    def forward(self, x):\n        # Initialize hidden state with zeros\n        h0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).requires_grad_()\n\n        # One time step\n        # We need to detach the hidden state to prevent exploding/vanishing gradients\n        # This is part of truncated backpropagation through time (BPTT)\n        out, hn = self.rnn(x, h0.detach())\n\n        # Index hidden state of last time step\n        # out.size() --&gt; 100, 28, 100\n        # out[:, -1, :] --&gt; 100, 100 --&gt; just want last time step hidden states! \n        out = self.fc(out[:, -1, :]) \n        # out.size() --&gt; 100, 10\n        return out\n\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28\nhidden_dim = 100\nlayer_dim = 2  # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER\noutput_dim = 10\n\nmodel = RNNModel(input_dim, hidden_dim, layer_dim, output_dim)\n\n# JUST PRINTING MODEL &amp; PARAMETERS \nprint(model)\nprint(len(list(model.parameters())))\nfor i in range(len(list(model.parameters()))):\n    print(list(model.parameters())[i].size())\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)  \n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\n\n# Number of steps to unroll\nseq_dim = 28  \n\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as tensors with gradient accumulation abilities\n        images = images.view(-1, seq_dim, input_dim).requires_grad_()\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        # outputs.size() --&gt; 100, 10\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                # Resize images\n                images = images.view(-1, seq_dim, input_dim)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre></p> <pre><code>RNNModel(\n  (rnn): RNN(28, 100, num_layers=2, batch_first=True)\n  (fc): Linear(in_features=100, out_features=10, bias=True)\n)\n\n10\n\ntorch.Size([100, 28])\ntorch.Size([100, 100])\ntorch.Size([100])\ntorch.Size([100])\ntorch.Size([100, 100])\ntorch.Size([100, 100])\ntorch.Size([100])\ntorch.Size([100])\ntorch.Size([10, 100])\ntorch.Size([10])\nIteration: 500. Loss: 0.5943437218666077. Accuracy: 77\nIteration: 1000. Loss: 0.22048641741275787. Accuracy: 91\nIteration: 1500. Loss: 0.18479223549365997. Accuracy: 94\nIteration: 2000. Loss: 0.2723771929740906. Accuracy: 91\nIteration: 2500. Loss: 0.18817797303199768. Accuracy: 92\nIteration: 3000. Loss: 0.1685929149389267. Accuracy: 92\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#summary-of-results","title":"Summary of Results","text":"Model A Model B Model C ReLU ReLU Tanh 1 Hidden Layer 2 Hidden Layers 2 Hidden Layers 100 Hidden Units 100 Hidden Units 100 Hidden Units 92.48% 95.09% 95.54%"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#general-deep-learning-notes","title":"General Deep Learning Notes","text":"<ul> <li>2 ways to expand a recurrent neural network<ul> <li>More non-linear activation units (neurons)</li> <li>More hidden layers</li> </ul> </li> <li>Cons<ul> <li>Need a larger dataset<ul> <li>Curse of dimensionality</li> </ul> </li> <li>Does not necessarily mean higher accuracy</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#3-building-a-recurrent-neural-network-with-pytorch-gpu","title":"3. Building a Recurrent Neural Network with PyTorch (GPU)","text":""},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#model-c-2-hidden-layer-tanh","title":"Model C: 2 Hidden Layer (Tanh)","text":"<p>GPU: 2 things must be on GPU - <code>model</code> - <code>tensors</code></p>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#steps_3","title":"Steps","text":"<ul> <li>Step 1: Load Dataset</li> <li>Step 2: Make Dataset Iterable</li> <li>Step 3: Create Model Class</li> <li>Step 4: Instantiate Model Class</li> <li>Step 5: Instantiate Loss Class</li> <li>Step 6: Instantiate Optimizer Class</li> <li>Step 7: Train Model</li> </ul> <p>2 Layer RNN + Tanh</p> <pre><code>import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\n\n'''\nSTEP 1: LOADING DATASET\n'''\ntrain_dataset = dsets.MNIST(root='./data', \n                            train=True, \n                            transform=transforms.ToTensor(),\n                            download=True)\n\ntest_dataset = dsets.MNIST(root='./data', \n                           train=False, \n                           transform=transforms.ToTensor())\n\n'''\nSTEP 2: MAKING DATASET ITERABLE\n'''\n\nbatch_size = 100\nn_iters = 3000\nnum_epochs = n_iters / (len(train_dataset) / batch_size)\nnum_epochs = int(num_epochs)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n                                          batch_size=batch_size, \n                                          shuffle=False)\n\n'''\nSTEP 3: CREATE MODEL CLASS\n'''\n\nclass RNNModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):\n        super(RNNModel, self).__init__()\n        # Hidden dimensions\n        self.hidden_dim = hidden_dim\n\n        # Number of hidden layers\n        self.layer_dim = layer_dim\n\n        # Building your RNN\n        # batch_first=True causes input/output tensors to be of shape\n        # (batch_dim, seq_dim, feature_dim)\n        self.rnn = nn.RNN(input_dim, hidden_dim, layer_dim, batch_first=True, nonlinearity='tanh')\n\n        # Readout layer\n        self.fc = nn.Linear(hidden_dim, output_dim)\n\n    def forward(self, x):\n        # Initialize hidden state with zeros\n        #######################\n        #  USE GPU FOR MODEL  #\n        #######################\n        h0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).to(device)\n\n        # One time step\n        # We need to detach the hidden state to prevent exploding/vanishing gradients\n        # This is part of truncated backpropagation through time (BPTT)\n        out, hn = self.rnn(x, h0.detach())\n\n        # Index hidden state of last time step\n        # out.size() --&gt; 100, 28, 100\n        # out[:, -1, :] --&gt; 100, 100 --&gt; just want last time step hidden states! \n        out = self.fc(out[:, -1, :]) \n        # out.size() --&gt; 100, 10\n        return out\n\n'''\nSTEP 4: INSTANTIATE MODEL CLASS\n'''\ninput_dim = 28\nhidden_dim = 100\nlayer_dim = 2  # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER\noutput_dim = 10\n\nmodel = RNNModel(input_dim, hidden_dim, layer_dim, output_dim)\n\n#######################\n#  USE GPU FOR MODEL  #\n#######################\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\n\n'''\nSTEP 5: INSTANTIATE LOSS CLASS\n'''\ncriterion = nn.CrossEntropyLoss()\n\n'''\nSTEP 6: INSTANTIATE OPTIMIZER CLASS\n'''\nlearning_rate = 0.1\n\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)  \n\n'''\nSTEP 7: TRAIN THE MODEL\n'''\n\n# Number of steps to unroll\nseq_dim = 28  \n\niter = 0\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        # Load images as tensors with gradient accumulation abilities\n        #######################\n        #  USE GPU FOR MODEL  #\n        #######################\n        images = images.view(-1, seq_dim, input_dim).requires_grad_().to(device)\n        labels = labels.to(device)\n\n        # Clear gradients w.r.t. parameters\n        optimizer.zero_grad()\n\n        # Forward pass to get output/logits\n        # outputs.size() --&gt; 100, 10\n        outputs = model(images)\n\n        # Calculate Loss: softmax --&gt; cross entropy loss\n        loss = criterion(outputs, labels)\n\n        # Getting gradients w.r.t. parameters\n        loss.backward()\n\n        # Updating parameters\n        optimizer.step()\n\n        iter += 1\n\n        if iter % 500 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                #######################\n                #  USE GPU FOR MODEL  #\n                #######################\n                images = images.view(-1, seq_dim, input_dim).to(device)\n\n                # Forward pass only to get logits/output\n                outputs = model(images)\n\n                # Get predictions from the maximum value\n                _, predicted = torch.max(outputs.data, 1)\n\n                # Total number of labels\n                total += labels.size(0)\n\n                # Total correct predictions\n                #######################\n                #  USE GPU FOR MODEL  #\n                #######################\n                if torch.cuda.is_available():\n                    correct += (predicted.cpu() == labels.cpu()).sum()\n                else:\n                    correct += (predicted == labels).sum()\n\n            accuracy = 100 * correct / total\n\n            # Print Loss\n            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n</code></pre> <pre><code>Iteration: 500. Loss: 0.5983774662017822. Accuracy: 81\nIteration: 1000. Loss: 0.2960105836391449. Accuracy: 86\nIteration: 1500. Loss: 0.19428101181983948. Accuracy: 93\nIteration: 2000. Loss: 0.11918395012617111. Accuracy: 95\nIteration: 2500. Loss: 0.11246936023235321. Accuracy: 95\nIteration: 3000. Loss: 0.15849310159683228. Accuracy: 95\n</code></pre>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#summary","title":"Summary","text":"<p>We've learnt to...</p> <p>Success</p> <ul> <li> Feedforward Neural Networks Transition to Recurrent Neural Networks</li> <li> RNN Models in PyTorch<ul> <li> Model A: 1 Hidden Layer RNN (ReLU)</li> <li> Model B: 2 Hidden Layer RNN (ReLU)</li> <li> Model C: 2 Hidden Layer RNN (Tanh)</li> </ul> </li> <li> Models Variation in Code<ul> <li> Modifying only step 4</li> </ul> </li> <li> Ways to Expand Model\u2019s Capacity<ul> <li> More non-linear activation units (neurons)</li> <li> More hidden layers</li> </ul> </li> <li> Cons of Expanding Capacity<ul> <li> Need more data</li> <li> Does not necessarily mean higher accuracy</li> </ul> </li> <li> GPU Code<ul> <li> 2 things on GPU<ul> <li> model</li> <li> tensors with gradient accumulation abilities</li> </ul> </li> <li> Modifying only Step 3, 4 and 7</li> </ul> </li> <li> 7 Step Model Building Recap<ul> <li> Step 1: Load Dataset</li> <li> Step 2: Make Dataset Iterable</li> <li> Step 3: Create Model Class</li> <li> Step 4: Instantiate Model Class</li> <li> Step 5: Instantiate Loss Class</li> <li> Step 6: Instantiate Optimizer Class</li> <li> Step 7: Train Model</li> <li> Step 7: Train Model</li> </ul> </li> </ul>"},{"location":"deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/#citation","title":"Citation","text":"<p>If you have found these useful in your research, presentations, school work, projects or workshops, feel free to cite using this DOI.</p> <p></p>"},{"location":"deep_learning/production_pytorch/speed_optimization_basics_numba/","title":"Speed Optimization Basics: Numba","text":""},{"location":"deep_learning/production_pytorch/speed_optimization_basics_numba/#when-to-use-numba","title":"When to use Numba","text":"<ul> <li>Numba works well when the code relies a lot on (1) numpy, (2) loops, and/or (2) cuda.</li> <li>Hence, we would like to maximize the use of numba in our code where possible where there are loops/numpy</li> </ul>"},{"location":"deep_learning/production_pytorch/speed_optimization_basics_numba/#numba-cpu-nopython","title":"Numba CPU: nopython","text":"<ul> <li>For a basic numba application, we can cecorate python function thus allowing it to run without python interpreter</li> <li>Essentially, it will compile the function with specific arguments once into machine code, then uses the cache subsequently</li> </ul>"},{"location":"deep_learning/production_pytorch/speed_optimization_basics_numba/#with-numba-no-python","title":"With Numba: no python","text":"<pre><code>from numba import jit, prange\nimport numpy as np\n\n# Numpy array of 10k elements\ninput_ndarray = np.random.rand(10000).reshape(10000)\n\n# This is the only extra line of code you need to add\n# which is a decorator\n@jit(nopython=True)\ndef go_fast(a):\n    trace = 0\n    for i in range(a.shape[0]):\n        trace += np.tanh(a[i])\n    return a + trace\n\n%timeit go_fast(input_ndarray)\n</code></pre> <pre><code>161 \u00b5s \u00b1 2.62 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 10000 loops each)\n</code></pre>"},{"location":"deep_learning/production_pytorch/speed_optimization_basics_numba/#without-numba","title":"Without numba","text":"<ul> <li>This is much slower, time measured in the millisecond space rather than microsecond with <code>@jit(nopython=True)</code> or <code>@njit</code></li> </ul> <pre><code># Without numba: notice how this is really slow\ndef go_normal(a):\n    trace = 0\n    for i in range(a.shape[0]):\n        trace += np.tanh(a[i])\n    return a + trace\n\n%timeit go_normal(input_ndarray)\n</code></pre> <pre><code>10.5 ms \u00b1 163 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 100 loops each)\n</code></pre>"},{"location":"deep_learning/production_pytorch/speed_optimization_basics_numba/#numba-cpu-parallel","title":"Numba CPU: parallel","text":"<ul> <li>Here, instead of the normal <code>range()</code> function we would use for loops, we would need to use <code>prange()</code> which allows us to execute the loops in parallel on separate threads</li> <li>As you can see, it's slightly faster than <code>@jit(nopython=True)</code></li> </ul> <pre><code>@jit(nopython=True, parallel=True)\ndef go_even_faster(a):\n    trace = 0\n    for i in prange(a.shape[0]):\n        trace += np.tanh(a[i])\n    return a + trace\n\n%timeit go_even_faster(input_ndarray)\n</code></pre> <pre><code>148 \u00b5s \u00b1 71.3 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1 loop each)\n</code></pre>"},{"location":"deep_learning/production_pytorch/speed_optimization_basics_numba/#numba-cpu-fastmath","title":"Numba CPU: fastmath","text":"<ul> <li>What if we relax our condition of strictly adhering to <code>IEEE 754</code>. </li> <li>We can have faster performance (depends)</li> <li>I would say this is the least additional speed-up unless you really dig into areas where <code>fastmath=True</code> thrives</li> </ul> <pre><code>@jit(nopython=True, parallel=True, fastmath=True)\ndef go_super_fast(a):\n    trace = 0\n    for i in prange(a.shape[0]):\n        trace += np.tanh(a[i])\n    return a + trace\n\n%timeit go_super_fast(input_ndarray)\n</code></pre> <pre><code>113 \u00b5s \u00b1 39.7 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1 loop each)\n</code></pre>"},{"location":"deep_learning/production_pytorch/speed_optimization_basics_numba/#summary","title":"Summary","text":"<ul> <li>When to use Numba<ul> <li>(1) numpy array or torch tensors,</li> <li>(2) loops, and/or</li> <li>(3) cuda</li> </ul> </li> <li>Numba CPU: nopython\u00b6</li> <li>Numba CPU: parallel </li> <li>Numba CPU: fastmath</li> </ul>"},{"location":"language_model/intro/","title":"Language Model","text":"<p>We'll be covering both CPU and GPU implementations of Large Language Models (LLMs) and Multimodal Language Models.</p> Packages and Languages you will Learn to Use <ul> <li>Python</li> <li>PyTorch</li> <li>Llamaindex</li> <li>Ollama</li> <li>Huggingface</li> <li>Apptainer</li> </ul>"},{"location":"language_model/containers/hpc_containers_apptainer/","title":"Containers","text":"<p>To run LLMs easily from laptops and desktops to the cloud, we will be introducing Apptainer (formerly Singularity) to achieve this where it has largely been adopted by the community to execute HPC workloads locally and in HPC data centers.</p>"},{"location":"language_model/containers/hpc_containers_apptainer/#benefits-of-apptainer","title":"Benefits of Apptainer","text":"<p>Apptainer has quite a few benefits such as:</p> <ul> <li>Mobility of Compute: It supports computational mobility through a single-file SIF container format.</li> <li>Integration over Isolation: It emphasizes integration, utilizing GPUs, high-speed networks, and parallel filesystems by default. In particular, it provides native support for running application containers using NVIDIA's CUDA or AMD's ROCm, enabling easy access to GPU-enabled machine learning frameworks like PyTorch, irrespective of the host operating system, provided the host has a driver and library installation for CUDA/ROCm.</li> <li>Compatility with Docker: It is 100% OCI compatible and aims for maximum compatibility with Docker.</li> <li>Designed for Security: It allows unprivileged users to use containers while preventing privilege escalation within the container and this means that users are the same within and outside the container.</li> <li>Verifiable Reproducibility and Security: It ensures trustworthy reproducibility and security through cryptographic signatures and an immutable container image format.</li> </ul>"},{"location":"language_model/containers/hpc_containers_apptainer/#installing-apptainer","title":"Installing Apptainer","text":"<p>The following guide assumes a Linux distribution. We have also rigorously tested this on Windows WSL2 Ubuntu which works perfectly, so if you are on a Windows machine, this guide would work too. We are pleasantly surprised till this day, the sheer investment and improvement Windows made to bring WSL2 to a state where it is almost as good as a bare metal Linux distribution installation.</p>"},{"location":"language_model/containers/hpc_containers_apptainer/#install-apptainer","title":"Install Apptainer","text":"<p>All you need is to install Apptainer to be able to leverage on this repository to work in containers with multiple environments (CPU/GPU) with any packages and OS independent of your host (local) machine.</p>"},{"location":"language_model/containers/hpc_containers_apptainer/#install-nvidia-libraries","title":"Install NVIDIA libraries","text":"<p>I advise you to use the Lambda Stack to manage your drivers effectively without errors. It's a single bash script that allows you to install and upgrade your NVIDIA drivers.</p> <p>Check CUDA toolkit installation via <code>nvcc -V</code></p>"},{"location":"language_model/containers/hpc_containers_apptainer/#set-nvidia-container-cli-apptainer-paths-deprecated","title":"Set NVIDIA Container CLI Apptainer Paths (Deprecated)","text":"<p>Not necessary in Apptainer, only in Singularity.</p> <p>To get nvidia-container-cli path <pre><code>which nvidia-container-cli\n</code></pre></p> <p>To set <pre><code>sudo singularity config global --set \"nvidia-container-cli path\" \"/usr/bin/nvidia-container-cli\"\n</code></pre></p> <p>To check <pre><code>sudo singularity config global --get \"nvidia-container-cli path\"\n</code></pre></p>"},{"location":"language_model/containers/hpc_containers_apptainer/#how-to-use-apptainer-in-cpugpu-mode","title":"How to Use Apptainer in CPU/GPU Mode","text":""},{"location":"language_model/containers/hpc_containers_apptainer/#option-a-transparent-image-container-workflow","title":"Option A: Transparent Image Container Workflow","text":"<p>This is recommended to build apptainer containers as it's transparent with the greatest reproducibility.</p>"},{"location":"language_model/containers/hpc_containers_apptainer/#1-build-container-sif","title":"1. Build container (<code>.sif</code>)","text":"<p>To build and test apptainer image container, simply run the following command in any of the folders:</p> <pre><code>bash build_sif.sh\n</code></pre>"},{"location":"language_model/containers/hpc_containers_apptainer/#2-run-container","title":"2. Run container","text":"<p>For CPU <pre><code>apptainer shell apptainer_container_$VER_sandbox.sif\n</code></pre></p> <p>For GPU using NVIDIA Container CLI <pre><code>apptainer shell --nv --nvccli apptainer_container_$VER_sandbox.sif\n</code></pre></p> <p>A quick way to test if you're able have everything good to run is to run <code>nvidia-smi</code> when you shell into the container. </p>"},{"location":"language_model/containers/hpc_containers_apptainer/#notes-on-gpu-availability","title":"Notes on GPU Availability","text":"<p>If you do not have a GPU on your host or local machine, the image will still be built and can be shared and used by machines with GPUs! The test for CUDA will fail on your local machine without GPU which will just spit out an error that you do not have a driver. There's nothing to worry about that.</p>"},{"location":"language_model/containers/hpc_containers_apptainer/#option-b-blackbox","title":"Option B: Blackbox","text":"<p>This is not recommended to build production apptainer containers but it is good for experimentation to determine the right configuration to put into your definition file for <code>Option A</code>.</p> <p>Note, replace <code>$VER</code> with whatever version is in the specific folder.</p>"},{"location":"language_model/containers/hpc_containers_apptainer/#1-build-container-sif_1","title":"1. Build container (<code>.sif</code>)","text":"<p>To build and test apptainer image container, simply run the following command in any of the folders: <pre><code>bash build_sif.sh\n</code></pre></p>"},{"location":"language_model/containers/hpc_containers_apptainer/#2-convert-container-to-mutable-sandbox","title":"2. Convert container to mutable sandbox","text":"<pre><code>apptainer build --sandbox apptainer_container_$VER_sandbox apptainer_container_$VER.sif\n</code></pre>"},{"location":"language_model/containers/hpc_containers_apptainer/#3-shell-into-writable-sandbox","title":"3. Shell into writable sandbox","text":"<p>You can install new packages and make persisting changes to the container in this step for experimentation. <pre><code>apptainer shell --writable apptainer_container_$VER_sandbox\n</code></pre></p>"},{"location":"language_model/containers/hpc_containers_apptainer/#4-convert-container-to-immutable-image-sif","title":"4. Convert container to immutable image (<code>.sif</code>)","text":"<pre><code>apptainer build --sandbox apptainer_container_$VER_sandbox.sif apptainer_container_$VER_img\n</code></pre>"},{"location":"language_model/containers/hpc_containers_apptainer/#basic-apptainer-commands-cheatsheet","title":"Basic Apptainer Commands Cheatsheet","text":"<p>This provides a list of useful commands.</p> <pre><code># test apptainer\napptainer version\n\n# pull cowsay container from docker\napptainer pull docker://sylabsio/lolcow\n\n# build (generic that can also build from local definition files, it is like a swiss army knife, more flexible than pull)\napptainer build lolcow.sif docker://sylabsio/lolcow\n\n# run option 1 requiring to shell into the container\napptainer shell lolcow.sif\ncowsay moo\n\n# run option 2 without requiring to shell into the container\napptainer exec lolcow.sif cowsay moo\n\n# clean cache during builds\napptainer cache clean\n</code></pre>"},{"location":"language_model/containers/hpc_containers_apptainer/#available-containers-definition-files","title":"Available Containers &amp; Definition Files","text":"<p>We maintain a repository of Apptainer recipes in this repository, feel free to clone/download it locally.</p>"},{"location":"language_model/containers/hpc_containers_apptainer/#gpu-containers","title":"GPU Containers","text":"<p>GPU containers can be found in <code>./containers/gpu</code> when you clone the above repository.</p>"},{"location":"language_model/containers/hpc_containers_apptainer/#ollama-mistral-7b-workloads","title":"Ollama (Mistral 7b) Workloads","text":"<ul> <li>Go into container folder: <code>cd ./containers/gpu/ollama</code></li> <li>Run 1<sup>st</sup> session <code>apptainer shell --nv --nvccli apptainer_container_0.1.sif</code><ul> <li><code>ollama serve</code></li> </ul> </li> <li>Run 2<sup>nd</sup> session (another window) <code>apptainer shell --nv --nvccli apptainer.1.sif</code><ul> <li><code>ollama run mistral</code></li> <li>You can now communicate with mistral model in your bash, or any other model you can pull on ollama website</li> </ul> </li> </ul> <p>Model Choice</p> <p>This runs a Mistral model as an example. You can run any other models by swapping out <code>mistral</code> reference above to any models on Ollama's library.</p>"},{"location":"language_model/containers/hpc_containers_apptainer/#ollama-gemma-workloads","title":"Ollama (Gemma) Workloads","text":"<ul> <li>Go into container folder: <code>cd ./containers/gpu/ollama</code></li> <li>Run 1<sup>st</sup> session <code>apptainer shell --nv --nvccli apptainer_container_0.1.sif</code><ul> <li><code>ollama serve</code></li> </ul> </li> <li>Run 2<sup>nd</sup> session (another window) <code>apptainer shell --nv --nvccli apptainer.1.sif</code><ul> <li><code>ollama run gemma:7b</code> or <code>ollama run gemma:2b</code></li> <li>You can now communicate with gemma model in your bash, or any other model you can pull on ollama website</li> </ul> </li> </ul> <p>Model Choice</p> <p>This runs a Gemma 2b or 7b model as an example. You can run any other models by swapping out <code>gemma</code> reference above to any models on Ollama's library.</p>"},{"location":"language_model/containers/hpc_containers_apptainer/#llamaindex-workloads","title":"LLamaindex workloads","text":"<ul> <li>Go into container folder: <code>cd ./containers/gpu/llamaindex</code></li> <li>Run 1<sup>st</sup> session (first window)<code>apptainer shell --nv --nvccli apptainer_container_0.1.sif</code><ul> <li><code>ollama serve</code></li> </ul> </li> <li>Run 2<sup>nd</sup> session (second window) <code>apptainer shell --nv --nvccli apptainer_container_0.1.sif</code><ul> <li><code>ollama run mistral</code></li> </ul> </li> <li>Run 3<sup>rd</sup> session (third window) <code>apptainer shell --nv --nvccli apptainer_container_0.1.sif</code><ul> <li><code>python</code></li> <li><code>from llama_index.llms import Ollama</code></li> <li><code>llm = Ollama(model=\"mistral\")</code></li> <li><code>response = llm.complete(\"What is Singapore\")</code></li> <li><code>print(response)</code></li> </ul> </li> </ul> <p>Model Choice</p> <p>This runs a Mistral model as an example. You can run any other models by swapping out <code>mistral</code> reference above to any models on Ollama's library.</p>"},{"location":"language_model/containers/hpc_containers_apptainer/#cpu-containers","title":"CPU Containers","text":"<p>CPU containers can be found in <code>./containers/cpu</code> when you clone the above repository.</p>"},{"location":"language_model/containers/hpc_containers_apptainer/#math-workloads","title":"Math Workloads","text":"<ul> <li>Go into container folder: <code>cd ./containers/cpu/math</code><ul> <li>Run <code>apptainer shell apptainer_container_0.2.sif</code></li> </ul> </li> </ul>"},{"location":"language_model/containers/hpc_containers_apptainer/#summary","title":"Summary","text":"<p>We introduced Apptainer as a way to run CPU or GPU workloads that can scale from your desktops, to on-prem HPC data centers, or to cloud providers like Azure, AWS, and GCP. Subsequently, we introduced running LLMs within Apptainer offline locally with Gemma 2b/7b (Google) and Mistral 7b (Mistral AI) models. With portable container to run our code and models, we can now move to diving deep into LLMs, Multimodal Language Models, and Retrieval Augmented Generation (RAG).</p>"},{"location":"language_model/llm/llm_intro_hyperparameter_tuning/","title":"LLM Introduction &amp; Hyperparameter Tuning","text":"<p>In this tutorial, we will be covering LLMs leveraging on Ollama and LlamaIndex using Gemma:7b (Google) open-source model.</p>"},{"location":"language_model/llm/llm_intro_hyperparameter_tuning/#environment-setup","title":"Environment Setup","text":"<p>Follow our tutorial on Apptainer to get started. Once you have followed the tutorial till the Ollama section where you successfully ran <code>ollama serve</code> and <code>ollama run gemma:7b</code>, you can run the <code>apptainer shell --nv --nvccli apptainer_container_0.1.sif</code> command followed by <code>jupyter lab</code> to access and run this notebook.</p> <p>Directory Guide</p> <p>When you shell into the Apptainer <code>.sif</code> container, you will need to navigate the directory as you normally would into the Deep Learning Wizard repository that you cloned, requiring you to <code>cd ..</code> to go back a few directories and finally reaching the right folder. </p>"},{"location":"language_model/llm/llm_intro_hyperparameter_tuning/#question-and-answer-no-hyperparameter-tuning","title":"Question and Answer | No Hyperparameter Tuning","text":"<p>In this section, we will leverage on the <code>Gemma:7b</code> LLM model to ask basic questions to get responses.</p> <pre><code>import ollama\n</code></pre>"},{"location":"language_model/llm/llm_intro_hyperparameter_tuning/#question-1","title":"Question 1","text":"<pre><code># The 'chat' function is called with two parameters: 'model' and 'messages'.\nresponse = ollama.chat(\n    model='gemma:7b',  # The 'model' parameter specifies the model to be used. Here, 'gemma:7b' is the model.\n    messages=[  # The 'messages' parameter is a list of message objects.\n        {\n            'role': 'user',  # Each message object has a 'role' key. It can be 'user' or 'assistant'.\n            'content': 'What is Singapore?',  # The 'content' key contains the actual content of the message.\n        },\n    ]\n)\n\n# The 'chat' function returns a response object. \n# The content of the assistant's message is accessed using the keys 'message' and 'content'.\n# The 'print' function is used to display this content.\nprint(response['message']['content'])\n</code></pre> <pre><code>Singapore is a city-state located on the island of Singapore, a Southeast Asian island. It is a highly developed city known for its modern architecture, efficient transportation system, and vibrant cultural diversity.\n</code></pre>"},{"location":"language_model/llm/llm_intro_hyperparameter_tuning/#question-2","title":"Question 2","text":"<pre><code>response = ollama.chat(\n    model='gemma:7b',\n    messages=[\n        {\n            'role': 'user',\n            'content': 'What is a Large Language Model?',\n        },\n    ]\n)\n\nprint(response['message']['content'])\n</code></pre> <pre><code>Sure, a Large Language Model (LLM) is a type of language model that has been trained on a massive amount of text data and has the ability to engage in a wide range of natural language processing tasks. LLMs are typically designed to have a large number of parameters, which allows them to learn complex relationships between words and sentences. LLMs are often used for tasks such as text summarization, translation, and code generation.\n</code></pre> <p>In our second question, we change the question to \"What is a Large Language Model?\" and you can observe how the answer is slightly longer than the first question \"What is Singapore\". In the next section, you will discover that this relates to a few hyperparemeters in LLMs that can be tweaked.</p>"},{"location":"language_model/llm/llm_intro_hyperparameter_tuning/#question-and-answer-hyperparameter-tuning","title":"Question and Answer | Hyperparameter Tuning","text":""},{"location":"language_model/llm/llm_intro_hyperparameter_tuning/#temperature-tuning","title":"Temperature Tuning","text":"<p>The <code>temperature</code> parameter in LLMs plays a pivotal role in determining the predictability of the output. </p> <ul> <li> <p>Lower temperature values (e.g., 0.2) lead to more predictable and consistent responses, but may risk being overly constrained or repetitive.</p> </li> <li> <p>Higher temperature values (e.g., 1.0) introduce more randomness and diversity, but can result in less consistency and occasional incoherence. The choice of temperature value is a trade-off between consistency and variety, and should be tailored to the specific requirements of your task.</p> </li> </ul>"},{"location":"language_model/llm/llm_intro_hyperparameter_tuning/#low-temperature","title":"Low Temperature","text":"<pre><code># Create new model\n# Set the temperature\n# Higher values (e.g., 1.0) make the output more random, while lower values (e.g., 0.1) make it more deterministic\nmodelfile='''\nFROM gemma:7b\nPARAMETER temperature 0.1\n'''\nollama.create('gemma_low_temp', modelfile=modelfile)\n\n# Now you can use the new model with adjusted temperature\nresponse = ollama.chat(\n    model='gemma_low_temp',\n    messages=[\n        {\n            'role': 'user',\n            'content': 'Write a happy birthday message, I would like to send to my friend.',\n        },\n    ]\n)\n\nprint(response['message']['content'])\n</code></pre> <pre><code>Happy Birthday, [Friend's Name]! I hope your day is filled with joy, laughter, and happiness. May all your wishes come true! \ud83c\udf89\ud83c\udf82\n</code></pre> <pre><code># Run multiple times\nfor i in range(3):\n    response = ollama.chat(\n        model='gemma_low_temp',\n        messages=[\n            {\n                'role': 'user',\n                'content': 'Write a happy birthday message, I would like to send to my friend.',\n            },\n        ],\n    )\n\n    # Print \n    print('-'*10)\n    print(f'Response {i}') \n    print('-'*10)\n    print(response['message']['content'])\n</code></pre> <pre><code>----------\nResponse 0\n----------\nHappy Birthday, [Friend's Name]! I hope your day is filled with joy, laughter, and happiness. May all your wishes come true! \ud83c\udf89\ud83c\udf82\n----------\nResponse 1\n----------\nHappy Birthday, [Friend's Name]! I hope your day is filled with joy, laughter, and happiness. May all your wishes come true! \ud83c\udf89\ud83c\udf82\n----------\nResponse 2\n----------\nHappy Birthday, [Friend's Name]! I hope your day is filled with joy, laughter, and happiness. May all your wishes come true! \ud83c\udf89\ud83c\udf82\n</code></pre> <p>We can see above it is the exact same answer calling the LLM 3 times.</p>"},{"location":"language_model/llm/llm_intro_hyperparameter_tuning/#high-temperature","title":"High Temperature","text":"<pre><code># Create new model\n# Set the temperature\n# Higher values (e.g., 1.0) make the output more random, while lower values (e.g., 0.1) make it more deterministic\nmodelfile='''\nFROM gemma:7b\nPARAMETER temperature 1.0\n'''\nollama.create('gemma_high_temp', modelfile=modelfile)\n\n# Now you can use the new model with adjusted temperature\nresponse = ollama.chat(\n    model='gemma_high_temp',\n    messages=[\n        {\n            'role': 'user',\n            'content': 'Write a happy birthday message, I would like to send to my friend.',\n        },\n    ]\n)\n\nprint(response['message']['content'])\n</code></pre> <pre><code>Happy Birthday, [Friend's Name]! I hope your day is filled with joy, laughter, and happiness! \ud83c\udf89\ud83c\udf82 \ud83c\udf89\n</code></pre> <pre><code># Run multiple times\nfor i in range(3):\n    response = ollama.chat(\n        model='gemma_high_temp',\n        messages=[\n            {\n                'role': 'user',\n                'content': 'Write a happy birthday message, I would like to send to my friend.',\n            },\n        ],\n    )\n\n    # Print \n    print('-'*10)\n    print(f'Response {i}') \n    print('-'*10)\n    print(response['message']['content'])\n</code></pre> <pre><code>----------\nResponse 0\n----------\nHappy Birthday, [Friend's Name]! I hope your day is filled with joy, laughter, and happiness. Let's celebrate your special day together!\n----------\nResponse 1\n----------\nHappy Birthday, [Friend's Name]! I hope your day is filled with joy, laughter, and happiness. Wishing you a very special day filled with memorable moments and sweet treats!\n----------\nResponse 2\n----------\nSure, here is a happy birthday message you can send to your friend:\n\n**Happy Birthday, [Friend's Name]!**\n\nMay your day be filled with joy, laughter, and happiness. I hope your special day is filled with everything you wish for. I'm sending you positive vibes and can't wait to see you soon.\n</code></pre> <p>Here, you can see very varied answer in each of the 3 calls to the LLM compared to lower temperature.</p>"},{"location":"language_model/llm/llm_intro_hyperparameter_tuning/#mathematical-interpretation","title":"Mathematical Interpretation","text":"<p>In LLMs, the <code>temperature</code> parameter is used to control the randomness of predictions by scaling the logits before applying soft(arg)max.</p> <ul> <li> <p>The model computes a score (also known as logits) for each possible next token based on the current context. These scores are then transformed into probabilities using the soft(arg)max function.</p> </li> <li> <p>The soft(arg)max function is defined as follows:</p> \\[\\text{softargmax}(x_i) = \\frac{e^{x_i}}{\\sum_i^j e^{x_i}}\\] </li> <li> <p>Before applying soft(arg)max, the logits are divided by the <code>temperature</code> value. This process is called temperature scaling and the equation becomes:</p> \\[\\text{softargmax}(x_i) = \\frac{e^{x_i/T}}{\\sum_i^j e^{x_i/T}}\\] </li> <li> <p>When <code>T</code> &gt; 1, it makes the distribution more uniform (increases randomness). When <code>T</code> &lt; 1, it makes the distribution more peaky (reduces randomness).</p> </li> </ul> <p>So, in simple terms, a higher temperature value makes the model's output more random, and a lower temperature makes it more deterministic.</p> <pre><code>import numpy as np\n\ndef softargmax(x, T=1.0):\n    e_x = np.exp(x / T)\n    return e_x / e_x.sum()\n\n# Define logits\nlogits = np.array([0.2, 0.3, 0.1, 0.4])\n\n# Compute soft(arg)max for different temperatures\nfor T in [0.1, 1.0]:\n    print(f\"Temperature: {T}\")\n    print(softargmax(logits, T=T))\n    print()\n</code></pre> <pre><code>Temperature: 0.1\n[0.08714432 0.23688282 0.0320586  0.64391426]\n\nTemperature: 1.0\n[0.23632778 0.26118259 0.21383822 0.28865141]\n</code></pre> <p>In the Python code above leveraging on <code>numpy</code> library, you can see that</p> <ul> <li><code>softargmax</code> is a function that computes the soft(arg)max of an array of logits <code>x</code> for a given temperature <code>T</code>.</li> <li>We define an array of logits and compute the soft(arg)max for different temperatures.</li> <li> <p>When you run this code, you'll see that as the temperature increases, the soft(arg)max output becomes more uniform (i.e., the probabilities are more evenly distributed), and as the temperature decreases, the soft(arg)max output becomes more peaky (i.e., one probability dominates the others). This illustrates how temperature can control the randomness of the model's output.</p> </li> <li> <p>To close this off, taking the max of the soft(arg)max output, you will observe how it gets more random in the max value as the soft(arg)max output becomes more uniform. This links to the concept of how the next word gets more random because of the max of the uniformity of the soft(arg)max output.</p> </li> </ul>"},{"location":"language_model/llm/llm_intro_hyperparameter_tuning/#top-k-tuning","title":"Top-K Tuning","text":"<p>In LLMs, the <code>top_k</code> hyperparameter is a key factor that influences the unpredictability of the generated output.</p> <ul> <li> <p>For smaller <code>top_k</code> values: The model behaves in a more predictable manner. It only takes into account a limited set of the most probable next tokens at each step of the generation process. This can result in responses that are more concise and consistent, but there\u2019s a possibility that the output may be too restricted or repetitive.</p> </li> <li> <p>For larger <code>top_k</code> values: The model takes into consideration a broader set of potential next tokens. This infuses more variety and randomness into the generated output. However, the responses can become less consistent and may occasionally be less coherent or pertinent. Therefore, the selection of the top_k value can be viewed as a balance between consistency and variety in the model\u2019s responses. It\u2019s crucial to adjust this parameter based on the specific needs of your task. </p> </li> </ul>"},{"location":"language_model/llm/llm_intro_hyperparameter_tuning/#low-k","title":"Low K","text":"<pre><code># Create new model\n\n# Set the temperature\n# Higher values (e.g., 1.0) make the output more random, while lower values (e.g., 0.1) make it more deterministic\n\n# Set the top_k\n# This parameter controls the number of tokens considered for each step of the generation process\nmodelfile='''\nFROM gemma:7b\nPARAMETER temperature 0.5\nPARAMETER top_k 3\n'''\nollama.create('gemma_topk_3', modelfile=modelfile)\n\n# Now you can use the new model with adjusted temperature\nresponse = ollama.chat(\n    model='gemma_topk_3',\n    messages=[\n        {\n            'role': 'user',\n            'content': 'Write a happy birthday message, I would like to send to my friend.',\n        },\n    ]\n)\n\nprint(response['message']['content'])\n</code></pre> <pre><code>Happy Birthday, [Friend's Name]! I hope your day is filled with joy, laughter, and happiness. May all your wishes come true!\n</code></pre> <pre><code># Run multiple times\nfor i in range(3):\n    response = ollama.chat(\n        model='gemma_topk_3',\n        messages=[\n            {\n                'role': 'user',\n                'content': 'Write a happy birthday message, I would like to send to my friend.',\n            },\n        ],\n    )\n\n    # Print \n    print('-'*10)\n    print(f'Response {i}') \n    print('-'*10)\n    print(response['message']['content'])\n</code></pre> <pre><code>----------\nResponse 0\n----------\nSure, here's a happy birthday message for your friend:\n\n**Happy Birthday, [Friend's Name]!**\n\nI hope your day is filled with joy, laughter, and happiness. May all your wishes come true. Have a blast! \ud83c\udf89\ud83c\udf82\ud83c\udf89\n----------\nResponse 1\n----------\nHappy Birthday, [Friend's Name]! I hope your day is filled with joy, laughter, and happiness. May all your wishes come true. \ud83c\udf89\ud83c\udf82\n----------\nResponse 2\n----------\nHappy Birthday, [Friend's Name]! I hope your day is filled with joy, laughter, and happiness. May all your wishes come true. \ud83c\udf89\ud83c\udf82\n</code></pre>"},{"location":"language_model/llm/llm_intro_hyperparameter_tuning/#high-k","title":"High K","text":"<pre><code># Create new model\n\n# Set the temperature\n# Higher values (e.g., 1.0) make the output more random, while lower values (e.g., 0.1) make it more deterministic\n\n# Set the top_k\n# This parameter controls the number of tokens considered for each step of the generation process\nmodelfile='''\nFROM gemma:7b\nPARAMETER temperature 0.5\nPARAMETER top_k 200\n'''\nollama.create('gemma_topk_200', modelfile=modelfile)\n\n# Now you can use the new model with adjusted temperature\nresponse = ollama.chat(\n    model='gemma_topk_200',\n    messages=[\n        {\n            'role': 'user',\n            'content': 'Write a happy birthday message, I would like to send to my friend.',\n        },\n    ]\n)\n\nprint(response['message']['content'])\n</code></pre> <pre><code>Happy Birthday, [Friend's name]! I hope your day is filled with joy, laughter, and happiness! \ud83c\udf89\ud83c\udf82\n</code></pre> <pre><code># Run multiple times\nfor i in range(3):\n    response = ollama.chat(\n        model='gemma_topk_200',\n        messages=[\n            {\n                'role': 'user',\n                'content': 'Write a happy birthday message, I would like to send to my friend.',\n            },\n        ],\n    )\n\n    # Print \n    print('-'*10)\n    print(f'Response {i}') \n    print('-'*10)\n    print(response['message']['content'])\n</code></pre> <pre><code>----------\nResponse 0\n----------\nSure, here is a happy birthday message for your friend:\n\n**Happy Birthday, [Friend's Name]!**\n\nI hope your day is filled with joy, laughter, and happiness. May all your wishes come true today.\n\nHave a wonderful birthday, and I look forward to seeing you soon.\n\n**Best regards,**\n\n[Your Name]\n----------\nResponse 1\n----------\nSure, here's a happy birthday message for your friend:\n\n**Happy Birthday, [Friend's Name]!**\n\nMay your day be filled with joy, laughter, and happiness. I hope your special day is filled with all your favorite things and that your wishes come true.\n\n**Have a wonderful birthday, my dear friend!**\n----------\nResponse 2\n----------\nSure, here's a happy birthday message for your friend:\n\n**Happy Birthday, [Friend's Name]!**\n\nMay your day be filled with joy, laughter, and happiness. I hope you have a wonderful time celebrating your special day!\n</code></pre> <p>You can observe that the reply is more diverse with a high <code>top_k</code> hyperparameter.</p>"},{"location":"language_model/llm/llm_intro_hyperparameter_tuning/#mathematical-interpretation_1","title":"Mathematical Interpretation","text":"<p>In LLMs, the <code>top_k</code> parameter is used to limit the number of next tokens considered for generation.</p> <ul> <li> <p>After computing the soft(arg)max probabilities for all possible next tokens, the model sorts these probabilities in descending order.</p> </li> <li> <p>The model then only considers the <code>top_k</code> tokens with the highest probabilities for the next step of the generation process.</p> </li> <li> <p>This process is called <code>top_k</code> sampling.</p> </li> </ul> <p>Here's a simple Python code snippet that illustrates how <code>top_k</code> works.</p> <pre><code>def top_k(logits, k):\n    # Sort the logits\n    sorted_indices = np.argsort(logits)\n\n    # Consider only the top k\n    top_k_indices = sorted_indices[-k:]\n\n    # Create a new array with only the top k probabilities\n    top_k_logits = logits[top_k_indices]\n\n    return top_k_logits\n\n# Define logits\nlogits = np.array([0.2, 0.3, 0.1, 0.4])\n\n# Compute top_k for different values of k\nfor k in [2, 3, 4]:\n    print(f\"Top {k} logits:\")\n    print(top_k(logits, k=k))\n    print()\n</code></pre> <pre><code>Top 2 logits:\n[0.3 0.4]\n\nTop 3 logits:\n[0.2 0.3 0.4]\n\nTop 4 logits:\n[0.1 0.2 0.3 0.4]\n</code></pre> <p>In the code above</p> <ul> <li> <p><code>top_k</code> is a function that computes the top <code>k</code> logits from an array of logits.</p> </li> <li> <p>We define an array of logits and compute the top <code>k</code> logits for different values of <code>k</code>.</p> </li> <li> <p>When you run this code, you'll see that as <code>k</code> increases, more logits are considered. This illustrates how <code>top_k</code> can control the number of tokens considered by the model.</p> </li> </ul>"},{"location":"language_model/llm/llm_intro_hyperparameter_tuning/#summary","title":"Summary","text":"<p>We covered the functionality of a basic LLM without any hyperparameter tuning. We then covered Temperature and Top-K hyperparameter tuning. It is important to note that there are many hyperparameters that can be tuned, and we will update this tutorial to gradually include as many as we can.</p>"},{"location":"machine_learning/intro/","title":"Machine Learning","text":"<p>We'll be covering both CPU and GPU implementations of machine learning algorithms and data wrangling libraries. </p> Packages and Languages you will Learn to Use <ul> <li>Python</li> <li>RAPIDS cuDF</li> <li>RAPIDS cuML</li> <li>Scikit-learn</li> <li>Pandas</li> <li>NumPy</li> <li>CuPy</li> </ul> <p>Work in progress</p> <p>This open-source portion is still a work in progress. Stay tuned while we gradually upload our tutorials and notes. Feel free to contact Ritchie Ng if you would like to contribute via our Facebook page.</p>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/","title":"GFD: GPU Fractional Differencing","text":""},{"location":"machine_learning/gpu/gpu_fractional_differencing/#summary","title":"Summary","text":"<p>Typically we attempt to achieve some form of stationarity via a transformation on our time series through common methods including integer differencing. However, integer differencing unnecessarily removes too much memory to achieve stationarity. An alternative, fractional differencing, allows us to achieve stationarity while maintaining the maximum amount of memory compared to integer differencing. While existing CPU-based implementations are inefficient for running fractional differencing on many large-scale time series, our GPU-based implementation enables rapid fractional differencing.</p> <p>This tutorial walks you through our GPU implementation of fractional differencing (GFD). Also, this tutorial is a special collaboration between industry and academia led by me.</p>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#credits","title":"Credits","text":"<p>Before we start, I would like to thank and give credits for this particular post to: 1. NVIDIA (Ettikan, Chris and Nick), Amazon AWS, ensemblecap.ai, and NExT++ (NUS School of Computing) 2. Marcos Lopez de Prado  for his recent push on the use of fractional differencing of which this guide is based on. 3. Hosking for his paper in 1981 on fractional differencing.</p>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#links","title":"Links","text":"<ul> <li>Code Repository: https://github.com/ritchieng/fractional_differencing_gpu</li> <li>Presentation: https://www.researchgate.net/publication/335159299_GFD_GPU_Fractional_Differencing_for_Rapid_Large-scale_Stationarizing_of_Time_Series_Data_while_Minimizing_Memory_Loss</li> </ul>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#installation-of-libraries-optional","title":"Installation of Libraries (Optional)","text":"<ul> <li>These steps are for easily running RAPIDS in Google Colab. You can skip this section if you've RAPIDS installed.</li> </ul> <pre><code>import pynvml\n\npynvml.nvmlInit()\nhandle = pynvml.nvmlDeviceGetHandleByIndex(0)\ndevice_name = pynvml.nvmlDeviceGetName(handle)\n\nprint(f'Currently running on {device_name}')\n\nif device_name != b'Tesla T4':\n    raise Exception(\"\"\"\n    Unfortunately this instance does not have a T4 GPU.\n\n    Please make sure you've configured Colab to request a GPU instance type.\n\n    Sometimes Colab allocates a Tesla K80 instead of a T4. Resetting the instance.\n\n    If you get a K80 GPU, try Runtime -&gt; Reset all runtimes -&gt; Keep trying!\n  \"\"\")\nelse:\n    print('Woo! You got the right kind of GPU, a Tesla T4!')\n</code></pre> <pre><code>Currently running on b'Tesla T4'\nWoo! You got the right kind of GPU, a Tesla T4!\n</code></pre> <pre><code># intall miniconda\n!wget -c https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh\n!chmod +x Miniconda3-4.5.4-Linux-x86_64.sh\n!bash ./Miniconda3-4.5.4-Linux-x86_64.sh -b -f -p /usr/local\n\n# install RAPIDS packages\n!conda install -q -y --prefix /usr/local -c conda-forge \\\n  -c rapidsai-nightly/label/cuda10.0 -c nvidia/label/cuda10.0 \\\n  cudf cuml\n\n# set environment vars\nimport sys, os, shutil\nsys.path.append('/usr/local/lib/python3.6/site-packages/')\nos.environ['NUMBAPRO_NVVM'] = '/usr/local/cuda/nvvm/lib64/libnvvm.so'\nos.environ['NUMBAPRO_LIBDEVICE'] = '/usr/local/cuda/nvvm/libdevice/'\n\n# copy .so files to current working dir\nfor fn in ['libcudf.so', 'librmm.so']:\n    shutil.copy('/usr/local/lib/'+fn, os.getcwd())\n</code></pre> <pre><code>--2019-08-19 05:36:25--  https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh\nResolving repo.continuum.io (repo.continuum.io)... 104.18.200.79, 104.18.201.79, 2606:4700::6812:c84f, ...\nConnecting to repo.continuum.io (repo.continuum.io)|104.18.200.79|:443... connected.\nHTTP request sent, awaiting response... 416 Requested Range Not Satisfiable\n\n    The file is already fully retrieved; nothing to do.\n\nPREFIX=/usr/local\ninstalling: python-3.6.5-hc3d631a_2 ...\nPython 3.6.5 :: Anaconda, Inc.\ninstalling: ca-certificates-2018.03.07-0 ...\ninstalling: conda-env-2.6.0-h36134e3_1 ...\ninstalling: libgcc-ng-7.2.0-hdf63c60_3 ...\ninstalling: libstdcxx-ng-7.2.0-hdf63c60_3 ...\ninstalling: libffi-3.2.1-hd88cf55_4 ...\ninstalling: ncurses-6.1-hf484d3e_0 ...\ninstalling: openssl-1.0.2o-h20670df_0 ...\ninstalling: tk-8.6.7-hc745277_3 ...\ninstalling: xz-5.2.4-h14c3975_4 ...\ninstalling: yaml-0.1.7-had09818_2 ...\ninstalling: zlib-1.2.11-ha838bed_2 ...\ninstalling: libedit-3.1.20170329-h6b74fdf_2 ...\ninstalling: readline-7.0-ha6073c6_4 ...\ninstalling: sqlite-3.23.1-he433501_0 ...\ninstalling: asn1crypto-0.24.0-py36_0 ...\ninstalling: certifi-2018.4.16-py36_0 ...\ninstalling: chardet-3.0.4-py36h0f667ec_1 ...\ninstalling: idna-2.6-py36h82fb2a8_1 ...\ninstalling: pycosat-0.6.3-py36h0a5515d_0 ...\ninstalling: pycparser-2.18-py36hf9f622e_1 ...\ninstalling: pysocks-1.6.8-py36_0 ...\ninstalling: ruamel_yaml-0.15.37-py36h14c3975_2 ...\ninstalling: six-1.11.0-py36h372c433_1 ...\ninstalling: cffi-1.11.5-py36h9745a5d_0 ...\ninstalling: setuptools-39.2.0-py36_0 ...\ninstalling: cryptography-2.2.2-py36h14c3975_0 ...\ninstalling: wheel-0.31.1-py36_0 ...\ninstalling: pip-10.0.1-py36_0 ...\ninstalling: pyopenssl-18.0.0-py36_0 ...\ninstalling: urllib3-1.22-py36hbe7ace6_0 ...\ninstalling: requests-2.18.4-py36he2e5f8d_1 ...\ninstalling: conda-4.5.4-py36_0 ...\nunlinking: ca-certificates-2019.6.16-hecc5488_0\nunlinking: certifi-2019.6.16-py36_1\nunlinking: conda-4.7.11-py36_0\nunlinking: cryptography-2.7-py36h72c5cf5_0\nunlinking: libgcc-ng-9.1.0-hdf63c60_0\nunlinking: libstdcxx-ng-9.1.0-hdf63c60_0\nunlinking: openssl-1.1.1c-h516909a_0\nunlinking: python-3.6.7-h381d211_1004\nunlinking: sqlite-3.28.0-h8b20d00_0\nunlinking: tk-8.6.9-hed695b0_1002\ninstallation finished.\nWARNING:\n    You currently have a PYTHONPATH environment variable set. This may cause\n    unexpected behavior when running the Python interpreter in Miniconda3.\n    For best results, please verify that your PYTHONPATH only points to\n    directories of packages that are compatible with the Python interpreter\n    in Miniconda3: /usr/local\nSolving environment: ...working... done\n\n## Package Plan ##\n\n  environment location: /usr/local\n\n  added / updated specs: \n    - cudf\n    - cuml\n\n\nThe following packages will be UPDATED:\n\n    ca-certificates: 2018.03.07-0         --&gt; 2019.6.16-hecc5488_0 conda-forge\n    certifi:         2018.4.16-py36_0     --&gt; 2019.6.16-py36_1     conda-forge\n    conda:           4.5.4-py36_0         --&gt; 4.7.11-py36_0        conda-forge\n    cryptography:    2.2.2-py36h14c3975_0 --&gt; 2.7-py36h72c5cf5_0   conda-forge\n    libgcc-ng:       7.2.0-hdf63c60_3     --&gt; 9.1.0-hdf63c60_0                \n    libstdcxx-ng:    7.2.0-hdf63c60_3     --&gt; 9.1.0-hdf63c60_0                \n    openssl:         1.0.2o-h20670df_0    --&gt; 1.1.1c-h516909a_0    conda-forge\n    python:          3.6.5-hc3d631a_2     --&gt; 3.6.7-h381d211_1004  conda-forge\n    sqlite:          3.23.1-he433501_0    --&gt; 3.28.0-h8b20d00_0    conda-forge\n    tk:              8.6.7-hc745277_3     --&gt; 8.6.9-hed695b0_1002  conda-forge\n\nPreparing transaction: ...working... done\nVerifying transaction: ...working... done\nExecuting transaction: ...working... done\n</code></pre> <p>You would need to do a <code>Runtime &gt; Restart and Run All</code> on your Google Colab notebook at this step.</p>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#imports","title":"Imports","text":"<pre><code># Critical imports for GPU cuDF\nimport nvstrings, nvcategory, cudf\n</code></pre> <pre><code># Other imports\nimport numpy as np\nimport pandas as pd\nimport time\nimport pandas_datareader.data as web\nfrom datetime import datetime\nfrom matplotlib import pyplot as plt\nfrom numba import cuda\n</code></pre>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#plotting-style","title":"Plotting Style","text":"<pre><code># Display settings: just to make it slightly prettier\n# There'll be a separate post on beautiful plots\n%matplotlib inline\nfigsize=(25, 6)\nplt.style.use('fivethirtyeight')\n</code></pre>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#necessity-of-fractional-differencing","title":"Necessity of Fractional Differencing","text":"<ul> <li>Before we get into why fractional differencing is critical, let us inspect a simple time series pulled from the Fed's database: S&amp;P 500.</li> </ul>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#pulling-processing-plotting-spx-time-series","title":"Pulling, Processing &amp; Plotting SPX Time Series","text":"<pre><code># Read SPX data 2010-2019 from FED database: https://fred.stlouisfed.org/categories/94\nasset_name = 'SPX'\nstart = datetime(2010, 1, 1)\nend = datetime(2019, 8, 1)\ndf_raw = web.DataReader('SP500', 'fred', start, end)\n\n# Basic clean up data: dropna (there are other more robust treatments)\ndf_raw = df_raw.dropna()\ndf_raw.columns = [asset_name]\n\ndf_raw.plot(figsize=figsize)\nplt.title('S&amp;P 500 Absolute Levels');\n</code></pre>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#integer-differencing-1-day-returns-spx-time-series","title":"Integer Differencing (1 Day Returns) SPX Time Series","text":"<ul> <li>Traditionally we typically might difference our time series by one day (daily returns) or more to reach some form of stationarity via tests like ADF, KPSS, PP and more.</li> <li>However, these forms of integer differencing causes more information than needed to be lost.</li> </ul> <pre><code># One day returns calculation through differencing by 1\ndifferencing_factor = 1\ndf_daily_returns = (df_raw - df_raw.shift(differencing_factor)) / df_raw.shift(differencing_factor) * 100\ndf_daily_returns.plot(figsize=figsize)\nplt.title(f'{asset_name} Daily Returns via Differencing 1 Day');\n</code></pre>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#integer-differencing-causes-unnecessary-memory-loss","title":"Integer Differencing Causes Unnecessary Memory Loss","text":"<ul> <li>Because of this reason, we propose fractional differencing to achieve stationarity while minimizing memory loss.</li> </ul>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#basic-fractional-differencing","title":"Basic Fractional Differencing","text":"<p>Fractional differencing allows us to achieve stationarity while maintaining the maximum amount of memory compared to integer differencing.</p> <p>This was originally introduced in 1981 in his paper \u201cFractional Differencing\u201d by J. R. M. Hosking1  and subsequent work by others concentrated on fast and efficient implementations for fractional differentiation for continuous stochastic processes. Recently, fractional differencing was introduced for financial time series through the fixed window fractional differencing instead of the expanding window method by Marcos Lopez de Prado2.</p>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#fractional-differencing-weight-function-and-plot","title":"Fractional Differencing Weight Function and Plot","text":"<ul> <li>Weights formula: \\(w_k = -w_{k-1} \\frac{d - k + 1}{k}\\)</li> <li>Weight converges to zero: \\(w_k \\rightarrow 0\\)</li> </ul> <pre><code>def get_weights(d, num_k):\n    r\"\"\"Calculate weights ($w$) for each lag ($k$) through\n    $w_k = -w_{k-1} \\frac{d - k + 1}{k}$.\n\n    Args:\n        d (int): differencing value.\n        num_k (int): number of lags (typically length of timeseries) to calculate w.\n    \"\"\"\n    w_k = np.array([1])\n\n    for k in range(1, num_k):\n        w_k = np.append(w_k, -w_k[-1] * ((d - k + 1)) / k)\n\n    w_k = w_k.reshape(-1, 1) \n\n    return w_k\n\ndef plot_weights(range_d, num_k, num_d_interval):\n    r\"\"\"Plot weights ($w$) for each lag ($k$) for varying differencing value ($d$).\n\n    Args:\n        range_d (list): range of differencing values to plot.\n        num_k (int): number of lags (typically length of timeseries) to plot.\n        num_d_interval (int): number of d interval.\n    \"\"\"\n    # Get differencing values\n    interval = np.linspace(range_d[0], range_d[1], num_d_interval)\n\n    # Dataframe of lags (rows) x number of differencing intervals (columns)\n    df_wk = pd.DataFrame(np.zeros((num_k, num_d_interval)))\n\n    # Get weights array per differencing value\n    for i, d in enumerate(interval):\n        df_wk[i] = get_weights(d, num_k)\n\n    # Rename columns for legend\n    df_wk.columns = [round(x, 1) for x in interval]\n\n    # Plot\n    df_wk.plot(figsize=figsize)\n    plt.title('Lag weights ($w_k$) for various differencing values (d)')\n    plt.legend(title='Differencing value (d)')\n    plt.ylabel('Weights (w)')\n    plt.xlabel('Lag (k)')\n    plt.show()\n\n    # Return weights\n    return df_wk\n\ndf_wk = plot_weights(range_d=[0, 1], num_k=7, num_d_interval=6)\n</code></pre>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#fractional-differencing-weight-function-table","title":"Fractional Differencing Weight Function Table","text":"<pre><code># Dataframe for lag weights (w) for various differencing values (d)\ndf_wk\n</code></pre> 0.0 0.2 0.4 0.6 0.8 1.0 0 1.0 1.000000 1.000000 1.000000 1.000000 1.0 1 -0.0 -0.200000 -0.400000 -0.600000 -0.800000 -1.0 2 -0.0 -0.080000 -0.120000 -0.120000 -0.080000 0.0 3 -0.0 -0.048000 -0.064000 -0.056000 -0.032000 0.0 4 -0.0 -0.033600 -0.041600 -0.033600 -0.017600 0.0 5 -0.0 -0.025536 -0.029952 -0.022848 -0.011264 0.0 6 -0.0 -0.020429 -0.022963 -0.016755 -0.007885 0.0"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#fractional-differencing-value-calculation-toy-example","title":"Fractional Differencing Value Calculation Toy Example","text":"<ul> <li>Assume spot have values 100, 99, 98, 97, 96, 95, 94 from k=0 to k=6</li> </ul>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#for-d-0-no-differencing","title":"For \\(d = 0\\) (no differencing)","text":"<pre><code>differenced_value = 100 * 1 + 99 * 0 + 98 * 0 + 97 * 0 + 96 * 0 + 95 * 0 + 94 * 0\nprint(f'Differenced value: {differenced_value}')\ndf_wk.iloc[:, 0]\n</code></pre> <pre><code>Differenced value: 100\n\n\n\n\n\n0    1.0\n1   -0.0\n2   -0.0\n3   -0.0\n4   -0.0\n5   -0.0\n6   -0.0\nName: 0.0, dtype: float64\n</code></pre>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#for-d-04-fractional-differencing","title":"For \\(d = 0.4\\) (fractional differencing)","text":"<pre><code>differenced_value = 100 * 1 + 99 * -0.400000 + 98 * -0.120000 + 97 * -0.064000 + 96 * -0.041600 + 95 * -0.041600 + 94 * -0.041600\nprint(f'Differenced value: {differenced_value}')\ndf_wk.iloc[:, 1]\n</code></pre> <pre><code>Differenced value: 30.576000000000004\n\n\n\n\n\n0    1.000000\n1   -0.200000\n2   -0.080000\n3   -0.048000\n4   -0.033600\n5   -0.025536\n6   -0.020429\nName: 0.2, dtype: float64\n</code></pre>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#for-d-1-integer-differencing","title":"For \\(d = 1\\) (integer differencing)","text":"<pre><code>differenced_value = 100 * 1 + 99 * -1 + 98 * 0 + 97 * 0 + 96 * 0 + 95 * 0 + 94 * 0\nprint(f'Differenced value: {differenced_value}')\ndf_wk.iloc[:, -1]\n</code></pre> <pre><code>Differenced value: 1\n\n\n\n\n\n0    1.0\n1   -1.0\n2    0.0\n3    0.0\n4    0.0\n5    0.0\n6    0.0\nName: 1.0, dtype: float64\n</code></pre>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#cumulative-sum-of-weights-behavior","title":"Cumulative Sum of Weights Behavior","text":"<ul> <li>Higher differencing value would accelerate cumulative sum weight decay</li> <li>This causes:<ol> <li>Less information taken from further lags</li> <li>More information loss from further lags</li> </ol> </li> </ul> <pre><code>df_wk.cumsum().plot(figsize=figsize)\nplt.title('Cumulative Sum of Lag weights ($\\sum_0^k w_k$) for various differencing values (d)')\nplt.legend(title='Differencing value ($d$)')\nplt.ylabel('Cumulative Sum of Weights ($\\sum_0^k w_k$)')\nplt.xlabel('Lag (k)');\n</code></pre>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#fixed-window-fractional-differencing-cpu","title":"Fixed Window Fractional Differencing (CPU)","text":""},{"location":"machine_learning/gpu/gpu_fractional_differencing/#floored-weights-function","title":"Floored Weights Function","text":"<ul> <li>For computational efficiency, we want to stop the calculation of weights when the weights are too small (below a certain small predetermined small value).</li> <li>Image we had a time series with 1,000,000 data points, we do not want to compute the weights up till 1m lag for our latest data point!  <ul> <li>We can simply calculate the weights till they're too small which is what this function does.</li> </ul> </li> </ul> <pre><code>def get_weights_floored(d, num_k, floor=1e-3):\n    r\"\"\"Calculate weights ($w$) for each lag ($k$) through\n    $w_k = -w_{k-1} \\frac{d - k + 1}{k}$ provided weight above a minimum value\n    (floor) for the weights to prevent computation of weights for the entire\n    time series.\n\n    Args:\n        d (int): differencing value.\n        num_k (int): number of lags (typically length of timeseries) to calculate w.\n        floor (float): minimum value for the weights for computational efficiency.\n    \"\"\"\n    w_k = np.array([1])\n    k = 1\n\n    while k &lt; num_k:\n        w_k_latest = -w_k[-1] * ((d - k + 1)) / k\n        if abs(w_k_latest) &lt;= floor:\n            break\n\n        w_k = np.append(w_k, w_k_latest)\n\n        k += 1\n\n    w_k = w_k.reshape(-1, 1) \n\n    return w_k\n\n# Show large time series being stopped\n# Notice how max lag is at 9?\nd = 0.5\nnum_k = 1000\nweights = get_weights_floored(d=d, num_k=num_k)\npd.DataFrame(weights).plot(legend=False, figsize=figsize)\n\n# Plot\nplt.title(f'Lag weights ($w_k$) for differencing value $d={d}$')\nplt.ylabel('Weights (w)')\nplt.xlabel('Lag (k)');\n</code></pre>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#fixed-window-fractional-differencing-function-cpu","title":"Fixed Window Fractional Differencing Function (CPU)","text":"<ul> <li>Here, we use the new floored weights function compared to our normal  weights function</li> <li>Calculate our differenced values through the dot product of our transposed weights matrix and our original time series within that window<ul> <li>Keep repeating by shifting the window by one time step until the end of the time series</li> </ul> </li> </ul> <pre><code>def frac_diff(df, d, floor=1e-3):\n    r\"\"\"Fractionally difference time series via CPU.\n\n    Args:\n        df (pd.DataFrame): dataframe of raw time series values.\n        d (float): differencing value from 0 to 1 where &gt; 1 has no FD.\n        floor (float): minimum value of weights, ignoring anything smaller.\n    \"\"\"\n    # Get weights window\n    weights = get_weights_floored(d=d, num_k=len(df), floor=floor)\n    weights_window_size = len(weights)\n\n    # Reverse weights\n    weights = weights[::-1]\n\n    # Blank fractionally differenced series to be filled\n    df_fd = []\n\n    # Slide window of time series, to calculated fractionally differenced values\n    # per window\n    for idx in range(weights_window_size, df.shape[0]):\n        # Dot product of weights and original values\n        # to get fractionally differenced values\n        date_idx = df.index[idx]\n        df_fd.append(np.dot(weights.T, df.iloc[idx - weights_window_size:idx]).item())\n\n    # Return FD values and weights\n    df_fd = pd.DataFrame(df_fd)\n\n    return df_fd, weights\n\n# Start timer\nstart = time.time()\n\ndf_raw_fd, weights = frac_diff(df_raw, d=0.5, floor=5e-5)\n\n# End timer\nend = time.time()\n\nprint(f'Time {end-start} s')\n\n# Plot\ndf_raw_fd.plot(figsize=figsize);\n</code></pre> <pre><code>Time 0.5417799949645996 s\n</code></pre> <pre><code>import multiprocessing\nmultiprocessing.cpu_count()\n</code></pre> <pre><code>4\n</code></pre> <p>Existing CPU-based implementations are inefficient for running fractional differencing on many large-scale time-series. GPU-based implementations provide an avenue to adapt to this century\u2019s big data requirements.</p>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#fixed-window-fractional-differencing-function-gpu","title":"Fixed Window Fractional Differencing Function (GPU)","text":"<ul> <li>This gives a slight 1.5x to 2x speed-up on an NVIDIA T4 on Google Colab for this tiny dataset<ul> <li>For larger time series in line with real-world situations,  it can easily be up to 100x to 1000x dataset (shown below).</li> </ul> </li> </ul> <pre><code>def moving_dot_product_kernel(in_data, out, window_size, weights):\n    # Set the first window_size-1 rows in each chunk to np.nan due \n    # insufficient history\n    for i in range(cuda.threadIdx.x, window_size - 1, cuda.blockDim.x):\n        out[i] = np.nan\n\n    # Compute dot product of preceding window_size rows\n    for i in range(cuda.threadIdx.x + window_size - 1, in_data.size, cuda.blockDim.x):\n        rolling_dot_product = 0.0\n\n        k = 0\n        for j in range(i - window_size + 1, i + 1):\n            rolling_dot_product += in_data[j] * weights[k][0]\n            k += 1\n\n        out[i] = rolling_dot_product \n\ndef frac_diff_gpu(df, d, floor=1e-3):\n    r\"\"\"Fractionally difference time series via GPU.\n\n    Args:\n        df (pd.DataFrame): dataframe of raw time series values.\n        d (float): differencing value from 0 to 1 where &gt; 1 has no FD.\n        floor (float): minimum value of weights, ignoring anything smaller.\n    \"\"\"\n\n    # Bring dataframe to GPU, reset index for GPU dot product kernel\n    gdf_raw = cudf.from_pandas(df).reset_index(drop=True)\n    gdf_raw.columns = ['in_data']\n\n    # Get weights window\n    weights = get_weights_floored(d=d, num_k=len(gdf_raw), floor=floor)\n    weights_window_size = len(weights)\n\n    # Reverse weights and as contiguous\n    weights = np.ascontiguousarray(weights[::-1])\n\n    # Bring weights to GPU\n    gdf_weights = cudf.DataFrame()\n    gdf_weights[gdf_raw.columns[0]] = weights.reshape(-1)\n\n    # Length of data\n    data_length = len(gdf_raw)\n\n    # T4: max of 518 threads per block.\n    # V100: max 1024 threads per block\n    threads_per_block = 518\n\n    # Chunk size split\n    # This has to be improved, but as a v0.1, it's sufficient to show speed-up\n    # Up to easily 100 million data points\n    trunk_size = data_length\n\n    # Get fractionally differenced time series through GPU function\n    gdf_raw_fd = gdf_raw.apply_chunks(moving_dot_product_kernel,\n                                 incols=['in_data'],\n                                 outcols=dict(out=np.float64),\n                                 kwargs=dict(window_size=weights_window_size, weights=weights),\n                                 chunks=list(range(0, data_length, trunk_size)) + [data_length],\n                                 tpb=threads_per_block)\n\n    # Bring to CPU for normal manipulation\n    df_raw_fd = gdf_raw_fd.to_pandas().dropna().iloc[:-1, 1]\n\n    return df_raw_fd, weights\n\n\n# Start timer\nstart = time.time()\n\ndf_raw_fd_from_gpu, weights = frac_diff_gpu(df_raw, d=0.5, floor=5e-5)\n\n# End timer\nend = time.time()\n\nprint(f'Time {end-start} s')\n\n# Plot\ndf_raw_fd_from_gpu.plot(figsize=figsize);\n</code></pre> <pre><code>Time 0.27262187004089355 s\n</code></pre>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#check-values","title":"Check Values","text":"<pre><code># Compare tail values\nprint('Tail values check')\nprint(df_raw_fd_from_gpu.tail().values)\nprint(df_raw_fd.tail().values.reshape(-1,))\n</code></pre> <pre><code>Tail values check\n[107.00340988 133.85520208 117.92878691 109.44132697  79.32562638]\n[107.00340988 133.85520208 117.92878691 109.44132697  79.32562638]\n</code></pre> <pre><code># Compare tail values\nprint('Head values check')\nprint(df_raw_fd_from_gpu.head().values)\nprint(df_raw_fd.head().values.reshape(-1,))\n</code></pre> <pre><code>Head values check\n[56.74989213 52.56766288 47.32421832 45.89772154 37.74401501]\n[56.74989213 52.56766288 47.32421832 45.89772154 37.74401501]\n</code></pre> <pre><code>num_rows_true = (df_raw_fd.values.astype(np.float32) == df_raw_fd_from_gpu.values.astype(np.float32)).astype(int).sum()\ntotal_rows = df_raw_fd.shape[0]\n\nprint(f'Number of rows equal: {num_rows_true}')\nprint(f'Total number of rows: {total_rows}')\n</code></pre> <pre><code>Number of rows equal: 2093\nTotal number of rows: 2093\n</code></pre>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#stationarity","title":"Stationarity","text":"<ul> <li>\"A stationary time series is one whose statistical properties such as mean, variance, autocorrelation, etc. are all constant over time\", Robert Nau, Duke University</li> <li>Essentially what we are trying to do here with fractional differencing is to attempt to have a stationary time series without losing too much memory.</li> <li>There are many ways to check if a time series is stationary, but we will be using 1 test here (they're more, but it's not the purpose of this tutorial)<ul> <li>Augmented Dickey\u2013Fuller (ADF) test: check for unit root</li> </ul> </li> </ul>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#adf-test","title":"ADF Test","text":"<pre><code># Import adf/kpss\nfrom statsmodels.tsa.stattools import adfuller\nimport warnings\nwarnings.filterwarnings(\"ignore\")  # Ignore deprecations for cleaner output\n</code></pre>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#adf-raw-data-d0","title":"ADF: raw data (d=0)","text":"<pre><code># ADF on raw data\n\nprint('Stationarity Test for SPX Absolute Levels (d=0)')\nprint('-'*50)\n\nresult = adfuller(df_raw['SPX'], regression='c')\n\nprint(f't-stat \\n\\t{result[0]:.2f}')\nprint(f'p-value \\n\\t{result[1]:.2f}')\nprint(f'Critical Values')\nfor key, value in result[4].items():\n    print(f'\\t{key}: {value:.2f}')\n\n# test statistic &gt; test statistic (1%) AND p-value &gt; 0.01\n# Fail to reject the null hypothesis that there's unit root at the 1% significance level\n# ==&gt; Fail to reject H0 ==&gt; Unit root ==&gt; Non-stationary\n</code></pre> <pre><code>Stationarity Test for SPX Absolute Levels (d=0)\n--------------------------------------------------\nt-stat \n    -0.11\np-value \n    0.95\nCritical Values\n    1%: -3.43\n    5%: -2.86\n    10%: -2.57\n</code></pre>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#adf-daily-return-data-d1","title":"ADF: daily return data (d=1)","text":"<pre><code># ADF on 1 day returns (d=1)\n\nprint('Stationarity Test for SPX Daily Returns (d=1)')\nprint('-'*50)\nresult = adfuller(df_raw['SPX'].pct_change(1).dropna(), regression='c')\n\nprint(f't-stat \\n\\t{result[0]:.2f}')\nprint(f'p-value \\n\\t{result[1]:.2f}')\nprint(f'Critical Values')\nfor key, value in result[4].items():\n    print(f'\\t{key}: {value:.2f}')\n\n# test statistic &lt; test statistic (1%) AND p-value &lt; 0.01\n# Reject the null hypothesis that there's unit root at the 1% significance level\n# ==&gt; Reject H0 ==&gt; No unit root ==&gt; Stationary\n</code></pre> <pre><code>Stationarity Test for SPX Daily Returns (d=1)\n--------------------------------------------------\nt-stat \n    -11.12\np-value \n    0.00\nCritical Values\n    1%: -3.43\n    5%: -2.86\n    10%: -2.57\n</code></pre>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#adf-fractionally-differenced-data-d05","title":"ADF: fractionally differenced data (d=0.5)","text":"<pre><code># ADF on fractionally differenced values\n\nprint('Stationarity Test (ADF) for SPX Fractionally Differenced (d=0.5)')\nprint('-'*50)\n\n\ndf_raw_fd_from_gpu, weights = frac_diff_gpu(df_raw, d=0.5, floor=5e-5)\n\nresult = adfuller(df_raw_fd_from_gpu, regression='c')\n\nprint(f't-stat \\n\\t{result[0]:.2f}')\nprint(f'p-value \\n\\t{result[1]:.2f}')\nprint(f'Critical Values')\nfor key, value in result[4].items():\n    print(f'\\t{key}: {value:.2f}')\n\n# test statistic &lt; test statistic (1%) AND p-value &lt; 0.01\n# Reject the null hypothesis that there's unit root at the 1% significance level\n# ==&gt; Reject H0 ==&gt; No unit root ==&gt; Stationary\n</code></pre> <pre><code>Stationarity Test (ADF) for SPX Fractionally Differenced (d=0.5)\n--------------------------------------------------\nt-stat \n    -3.86\np-value \n    0.00\nCritical Values\n    1%: -3.43\n    5%: -2.86\n    10%: -2.57\n</code></pre>"},{"location":"machine_learning/gpu/gpu_fractional_differencing/#large-scale-rapid-fractional-differencing","title":"Large-scale Rapid Fractional Differencing","text":"<ul> <li>Here we test on a 100k, 1m, 10m and 100m datapoint dataframe.</li> </ul> <pre><code># Create 100m data points\nlarge_time_series_length = int(1e6)\nprint(large_time_series_length)\n</code></pre> <pre><code>1000000\n</code></pre> <pre><code># Start timer\nstart = time.time()\n\ndf_raw = pd.DataFrame(np.arange(large_time_series_length) * np.random.rand(large_time_series_length))\n\ndf_raw_fd_from_gpu, weights = frac_diff_gpu(df_raw, d=0.5, floor=5e-5)\n\n# End timer\nend = time.time()\n\nprint(f'Time {end-start} s')\n\n# Check\nprint('FD Shape', df_raw_fd_from_gpu.shape[0])\nprint('Correct Shape', df_raw.shape[0] - weights.shape[0])\nprint('Original Shape', df_raw.shape[0])\n</code></pre> <pre><code>Time 0.4671785831451416 s\nFD Shape 999682\nCorrect Shape 999682\nOriginal Shape 1000000\n</code></pre> <pre><code># Start timer\nstart = time.time()\n\n# Create 100m data points\ndf_raw = pd.DataFrame(np.arange(large_time_series_length) * np.random.rand(large_time_series_length))\n\ndf_raw_fd, weights = frac_diff(df_raw, d=0.5, floor=5e-5)\n\n# End timer\nend = time.time()\n\nprint(f'Time {end-start} s')\n\n# Check\nprint('FD Shape', df_raw_fd_from_gpu.shape[0])\nprint('Correct Shape', df_raw.shape[0] - weights.shape[0])\nprint('Original Shape', df_raw.shape[0])\n</code></pre> <pre><code>Time 128.29733324050903 s\nFD Shape 999682\nCorrect Shape 999682\nOriginal Shape 1000000\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/","title":"RAPIDS cuDF","text":""},{"location":"machine_learning/gpu/rapids_cudf/#environment-setup","title":"Environment Setup","text":""},{"location":"machine_learning/gpu/rapids_cudf/#check-version","title":"Check Version","text":""},{"location":"machine_learning/gpu/rapids_cudf/#python-version","title":"Python Version","text":"<pre><code># Check Python Version\n!python --version\n</code></pre> <pre><code>Python 3.8.16\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/#ubuntu-version","title":"Ubuntu Version","text":"<pre><code># Check Ubuntu Version\n!lsb_release -a\n</code></pre> <pre><code>No LSB modules are available.\nDistributor ID: Ubuntu\nDescription:    Ubuntu 18.04.6 LTS\nRelease:    18.04\nCodename:   bionic\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/#check-cuda-version","title":"Check CUDA Version","text":"<pre><code># Check CUDA/cuDNN Version\n!nvcc -V &amp;&amp; which nvcc\n</code></pre> <pre><code>nvcc: NVIDIA (R) Cuda compiler driver\nCopyright (c) 2005-2021 NVIDIA Corporation\nBuilt on Sun_Feb_14_21:12:58_PST_2021\nCuda compilation tools, release 11.2, V11.2.152\nBuild cuda_11.2.r11.2/compiler.29618528_0\n/usr/local/cuda/bin/nvcc\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/#check-gpu-version","title":"Check GPU Version","text":"<pre><code># Check GPU\n!nvidia-smi\n</code></pre> <pre><code>Wed Jan  4 19:14:22 2023       \n+-----------------------------------------------------------------------------+\n| NVIDIA-SMI 460.32.03    Driver Version: 460.32.03    CUDA Version: 11.2     |\n|-------------------------------+----------------------+----------------------+\n| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n|                               |                      |               MIG M. |\n|===============================+======================+======================|\n|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n| N/A   48C    P0    29W /  70W |      0MiB / 15109MiB |      0%      Default |\n|                               |                      |                  N/A |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes:                                                                  |\n|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n|        ID   ID                                                   Usage      |\n|=============================================================================|\n|  No running processes found                                                 |\n+-----------------------------------------------------------------------------+\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/#setup","title":"Setup:","text":"<p>This set up script:</p> <ol> <li>Checks to make sure that the GPU is RAPIDS compatible</li> <li>Installs the current stable version of RAPIDSAI's core libraries using pip, which are:</li> <li>cuDF</li> <li>cuML</li> <li>cuGraph</li> <li>xgboost</li> </ol> <p>This will complete in about 3-4 minutes</p> <p>Please use the RAPIDS Conda Colab Template notebook if you need to install any of RAPIDS Extended libraries, such as: - cuSpatial - cuSignal - cuxFilter - cuCIM</p> <p>OR - nightly versions of any library </p> <pre><code># This get the RAPIDS-Colab install files and test check your GPU.  Run this and the next cell only.\n# Please read the output of this cell.  If your Colab Instance is not RAPIDS compatible, it will warn you and give you remediation steps.\n!git clone https://github.com/rapidsai/rapidsai-csp-utils.git\n!python rapidsai-csp-utils/colab/pip-install.py\n</code></pre> <pre><code>Cloning into 'rapidsai-csp-utils'...\nremote: Enumerating objects: 328, done.\u001b[K\nremote: Counting objects: 100% (157/157), done.\u001b[K\nremote: Compressing objects: 100% (102/102), done.\u001b[K\nremote: Total 328 (delta 92), reused 98 (delta 55), pack-reused 171\u001b[K\nReceiving objects: 100% (328/328), 94.64 KiB | 18.93 MiB/s, done.\nResolving deltas: 100% (154/154), done.\nLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\nCollecting pynvml\n  Downloading pynvml-11.4.1-py3-none-any.whl (46 kB)\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 47.0/47.0 KB 6.1 MB/s eta 0:00:00\nInstalling collected packages: pynvml\nSuccessfully installed pynvml-11.4.1\n***********************************************************************\nWoo! Your instance has the right kind of GPU, a Tesla T4!\nWe will now install RAPIDS via pip!  Please stand by, should be quick...\n***********************************************************************\n\nLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/, https://pypi.ngc.nvidia.com\nCollecting cudf-cu11\n  Downloading https://developer.download.nvidia.com/compute/redist/cudf-cu11/cudf_cu11-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (442.8 MB)\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 442.8/442.8 MB 3.5 MB/s eta 0:00:00\nCollecting cuml-cu11\n  Downloading https://developer.download.nvidia.com/compute/redist/cuml-cu11/cuml_cu11-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1359.8 MB)\ntcmalloc: large alloc 1359798272 bytes == 0x3116000 @  0x7f53812b21e7 0x4d30a0 0x4d312c 0x5d6f4c 0x51edd1 0x51ef5b 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x5d8868 0x4997a2 0x55cd91 0x5d8941 0x49abe4 0x55cd91 0x5d8941 0x4997a2\ntcmalloc: large alloc 1699749888 bytes == 0x541e4000 @  0x7f53812b3615 0x5d6f4c 0x51edd1 0x51ef5b 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x5d8868 0x4997a2 0x55cd91 0x5d8941 0x49abe4 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941\ntcmalloc: large alloc 1359798272 bytes == 0x3116000 @  0x7f53812b21e7 0x4d30a0 0x5dede2 0x6758aa 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4fe318 0x5da092 0x62042c 0x5d8d8c 0x561f80 0x4fd2db 0x4997c7 0x4fd8b5 0x4997c7 0x4fd8b5 0x49abe4 0x4f5fe9 0x55e146 0x4f5fe9 0x55e146 0x4f5fe9 0x55e146 0x5d8868 0x5da092 0x587116\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.4/1.4 GB 1.3 MB/s eta 0:00:00\nCollecting cugraph-cu11\n  Downloading https://developer.download.nvidia.com/compute/redist/cugraph-cu11/cugraph_cu11-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1028.4 MB)\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.0/1.0 GB 1.9 MB/s eta 0:00:00\nRequirement already satisfied: numba&gt;=0.56.2 in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (0.56.4)\nRequirement already satisfied: numpy in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (1.21.6)\nCollecting ptxcompiler-cu11\n  Downloading https://developer.download.nvidia.com/compute/redist/ptxcompiler-cu11/ptxcompiler_cu11-0.7.0.post1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.8 MB)\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 8.8/8.8 MB 99.1 MB/s eta 0:00:00\nCollecting cuda-python&lt;12.0,&gt;=11.7.1\n  Downloading cuda_python-11.8.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (16.2 MB)\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 16.2/16.2 MB 77.6 MB/s eta 0:00:00\nRequirement already satisfied: pyarrow==9.0.0 in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (9.0.0)\nRequirement already satisfied: pandas&lt;1.6.0dev0,&gt;=1.0 in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (1.3.5)\nRequirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (4.4.0)\nRequirement already satisfied: cupy-cuda11x in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (11.0.0)\nRequirement already satisfied: cachetools in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (5.2.0)\nRequirement already satisfied: fsspec&gt;=0.6.0 in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (2022.11.0)\nCollecting protobuf&lt;3.21.0a0,&gt;=3.20.1\n  Downloading protobuf-3.20.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl (1.0 MB)\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.0/1.0 MB 46.4 MB/s eta 0:00:00\nCollecting rmm-cu11\n  Downloading https://developer.download.nvidia.com/compute/redist/rmm-cu11/rmm_cu11-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.8 MB)\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.8/1.8 MB 61.1 MB/s eta 0:00:00\nCollecting cubinlinker-cu11\n  Downloading https://developer.download.nvidia.com/compute/redist/cubinlinker-cu11/cubinlinker_cu11-0.3.0.post1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.8 MB)\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 8.8/8.8 MB 99.9 MB/s eta 0:00:00\nRequirement already satisfied: packaging in /usr/local/lib/python3.8/dist-packages (from cudf-cu11) (21.3)\nCollecting nvtx&gt;=0.2.1\n  Downloading nvtx-0.2.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (453 kB)\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 453.6/453.6 KB 28.4 MB/s eta 0:00:00\nRequirement already satisfied: seaborn in /usr/local/lib/python3.8/dist-packages (from cuml-cu11) (0.11.2)\nCollecting raft-dask-cu11\n  Downloading https://developer.download.nvidia.com/compute/redist/raft-dask-cu11/raft_dask_cu11-22.12.0.post1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (210.5 MB)\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 210.5/210.5 MB 6.7 MB/s eta 0:00:00\nRequirement already satisfied: scipy in /usr/local/lib/python3.8/dist-packages (from cuml-cu11) (1.7.3)\nCollecting treelite==3.0.1\n  Downloading treelite-3.0.1-py3-none-manylinux2014_x86_64.whl (864 kB)\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 864.6/864.6 KB 38.0 MB/s eta 0:00:00\nCollecting treelite-runtime==3.0.1\n  Downloading treelite_runtime-3.0.1-py3-none-manylinux2014_x86_64.whl (191 kB)\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 191.9/191.9 KB 25.3 MB/s eta 0:00:00\nCollecting dask-cudf-cu11\n  Downloading https://developer.download.nvidia.com/compute/redist/dask-cudf-cu11/dask_cudf_cu11-22.12.0.post1-py3-none-any.whl (76 kB)\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 76.6/76.6 KB 12.2 MB/s eta 0:00:00\nCollecting pylibraft-cu11\n  Downloading https://developer.download.nvidia.com/compute/redist/pylibraft-cu11/pylibraft_cu11-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (580.3 MB)\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 580.3/580.3 MB 3.2 MB/s eta 0:00:00\nCollecting pylibcugraph-cu11\n  Downloading https://developer.download.nvidia.com/compute/redist/pylibcugraph-cu11/pylibcugraph_cu11-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1627.2 MB)\ntcmalloc: large alloc 1627185152 bytes == 0x541e8000 @  0x7f53812b21e7 0x4d30a0 0x4d312c 0x5d6f4c 0x51edd1 0x51ef5b 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x5d8868 0x4997a2 0x55cd91 0x5d8941 0x49abe4 0x55cd91 0x5d8941 0x4997a2\ntcmalloc: large alloc 2033983488 bytes == 0xb51b6000 @  0x7f53812b3615 0x5d6f4c 0x51edd1 0x51ef5b 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x5d8868 0x4997a2 0x55cd91 0x5d8941 0x49abe4 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.6/1.6 GB 1.1 MB/s eta 0:00:00\nCollecting dask-cuda\n  Downloading dask_cuda-22.12.0-py3-none-any.whl (121 kB)\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 121.1/121.1 KB 17.5 MB/s eta 0:00:00\nRequirement already satisfied: cython in /usr/local/lib/python3.8/dist-packages (from cuda-python&lt;12.0,&gt;=11.7.1-&gt;cudf-cu11) (0.29.32)\nRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.8/dist-packages (from numba&gt;=0.56.2-&gt;cudf-cu11) (5.2.0)\nRequirement already satisfied: llvmlite&lt;0.40,&gt;=0.39.0dev0 in /usr/local/lib/python3.8/dist-packages (from numba&gt;=0.56.2-&gt;cudf-cu11) (0.39.1)\nRequirement already satisfied: setuptools in /usr/local/lib/python3.8/dist-packages (from numba&gt;=0.56.2-&gt;cudf-cu11) (57.4.0)\nRequirement already satisfied: python-dateutil&gt;=2.7.3 in /usr/local/lib/python3.8/dist-packages (from pandas&lt;1.6.0dev0,&gt;=1.0-&gt;cudf-cu11) (2.8.2)\nRequirement already satisfied: pytz&gt;=2017.3 in /usr/local/lib/python3.8/dist-packages (from pandas&lt;1.6.0dev0,&gt;=1.0-&gt;cudf-cu11) (2022.7)\nRequirement already satisfied: fastrlock&gt;=0.5 in /usr/local/lib/python3.8/dist-packages (from cupy-cuda11x-&gt;cudf-cu11) (0.8.1)\nRequirement already satisfied: zict&gt;=0.1.3 in /usr/local/lib/python3.8/dist-packages (from dask-cuda-&gt;cugraph-cu11) (2.2.0)\nCollecting distributed==2022.11.1\n  Downloading distributed-2022.11.1-py3-none-any.whl (923 kB)\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 923.4/923.4 KB 50.7 MB/s eta 0:00:00\nCollecting dask==2022.11.1\n  Downloading dask-2022.11.1-py3-none-any.whl (1.1 MB)\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.1/1.1 MB 52.6 MB/s eta 0:00:00\nRequirement already satisfied: pynvml&gt;=11.0.0 in /usr/local/lib/python3.8/dist-packages (from dask-cuda-&gt;cugraph-cu11) (11.4.1)\nRequirement already satisfied: click&gt;=7.0 in /usr/local/lib/python3.8/dist-packages (from dask==2022.11.1-&gt;dask-cuda-&gt;cugraph-cu11) (7.1.2)\nRequirement already satisfied: toolz&gt;=0.8.2 in /usr/local/lib/python3.8/dist-packages (from dask==2022.11.1-&gt;dask-cuda-&gt;cugraph-cu11) (0.12.0)\nRequirement already satisfied: partd&gt;=0.3.10 in /usr/local/lib/python3.8/dist-packages (from dask==2022.11.1-&gt;dask-cuda-&gt;cugraph-cu11) (1.3.0)\nRequirement already satisfied: cloudpickle&gt;=1.1.1 in /usr/local/lib/python3.8/dist-packages (from dask==2022.11.1-&gt;dask-cuda-&gt;cugraph-cu11) (1.5.0)\nRequirement already satisfied: pyyaml&gt;=5.3.1 in /usr/local/lib/python3.8/dist-packages (from dask==2022.11.1-&gt;dask-cuda-&gt;cugraph-cu11) (6.0)\nRequirement already satisfied: tornado&lt;6.2,&gt;=6.0.3 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1-&gt;dask-cuda-&gt;cugraph-cu11) (6.0.4)\nRequirement already satisfied: locket&gt;=1.0.0 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1-&gt;dask-cuda-&gt;cugraph-cu11) (1.0.0)\nRequirement already satisfied: tblib&gt;=1.6.0 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1-&gt;dask-cuda-&gt;cugraph-cu11) (1.7.0)\nRequirement already satisfied: jinja2 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1-&gt;dask-cuda-&gt;cugraph-cu11) (2.11.3)\nRequirement already satisfied: psutil&gt;=5.0 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1-&gt;dask-cuda-&gt;cugraph-cu11) (5.4.8)\nRequirement already satisfied: urllib3 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1-&gt;dask-cuda-&gt;cugraph-cu11) (1.24.3)\nRequirement already satisfied: sortedcontainers!=2.0.0,!=2.0.1 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1-&gt;dask-cuda-&gt;cugraph-cu11) (2.4.0)\nRequirement already satisfied: msgpack&gt;=0.6.0 in /usr/local/lib/python3.8/dist-packages (from distributed==2022.11.1-&gt;dask-cuda-&gt;cugraph-cu11) (1.0.4)\nRequirement already satisfied: pyparsing!=3.0.5,&gt;=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging-&gt;cudf-cu11) (3.0.9)\nRequirement already satisfied: joblib&gt;=0.11 in /usr/local/lib/python3.8/dist-packages (from raft-dask-cu11-&gt;cuml-cu11) (1.2.0)\nCollecting ucx-py-cu11\n  Downloading https://developer.download.nvidia.com/compute/redist/ucx-py-cu11/ucx_py_cu11-0.29.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.3 MB)\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 8.3/8.3 MB 72.8 MB/s eta 0:00:00\nRequirement already satisfied: matplotlib&gt;=2.2 in /usr/local/lib/python3.8/dist-packages (from seaborn-&gt;cuml-cu11) (3.2.2)\nRequirement already satisfied: cycler&gt;=0.10 in /usr/local/lib/python3.8/dist-packages (from matplotlib&gt;=2.2-&gt;seaborn-&gt;cuml-cu11) (0.11.0)\nRequirement already satisfied: kiwisolver&gt;=1.0.1 in /usr/local/lib/python3.8/dist-packages (from matplotlib&gt;=2.2-&gt;seaborn-&gt;cuml-cu11) (1.4.4)\nRequirement already satisfied: six&gt;=1.5 in /usr/local/lib/python3.8/dist-packages (from python-dateutil&gt;=2.7.3-&gt;pandas&lt;1.6.0dev0,&gt;=1.0-&gt;cudf-cu11) (1.15.0)\nRequirement already satisfied: heapdict in /usr/local/lib/python3.8/dist-packages (from zict&gt;=0.1.3-&gt;dask-cuda-&gt;cugraph-cu11) (1.0.1)\nRequirement already satisfied: zipp&gt;=0.5 in /usr/local/lib/python3.8/dist-packages (from importlib-metadata-&gt;numba&gt;=0.56.2-&gt;cudf-cu11) (3.11.0)\nRequirement already satisfied: MarkupSafe&gt;=0.23 in /usr/local/lib/python3.8/dist-packages (from jinja2-&gt;distributed==2022.11.1-&gt;dask-cuda-&gt;cugraph-cu11) (2.0.1)\nInstalling collected packages: ptxcompiler-cu11, nvtx, cubinlinker-cu11, ucx-py-cu11, protobuf, cuda-python, treelite-runtime, treelite, dask, rmm-cu11, distributed, pylibraft-cu11, dask-cuda, cudf-cu11, raft-dask-cu11, pylibcugraph-cu11, dask-cudf-cu11, cuml-cu11, cugraph-cu11\n  Attempting uninstall: protobuf\n    Found existing installation: protobuf 3.19.6\n    Uninstalling protobuf-3.19.6:\n      Successfully uninstalled protobuf-3.19.6\n  Attempting uninstall: dask\n    Found existing installation: dask 2022.2.1\n    Uninstalling dask-2022.2.1:\n      Successfully uninstalled dask-2022.2.1\n  Attempting uninstall: distributed\n    Found existing installation: distributed 2022.2.1\n    Uninstalling distributed-2022.2.1:\n      Successfully uninstalled distributed-2022.2.1\nERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\ntensorflow 2.9.2 requires protobuf&lt;3.20,&gt;=3.9.2, but you have protobuf 3.20.3 which is incompatible.\ntensorboard 2.9.1 requires protobuf&lt;3.20,&gt;=3.9.2, but you have protobuf 3.20.3 which is incompatible.\nSuccessfully installed cubinlinker-cu11-0.3.0.post1 cuda-python-11.8.1 cudf-cu11-22.12.0 cugraph-cu11-22.12.0 cuml-cu11-22.12.0 dask-2022.11.1 dask-cuda-22.12.0 dask-cudf-cu11-22.12.0.post1 distributed-2022.11.1 nvtx-0.2.5 protobuf-3.20.3 ptxcompiler-cu11-0.7.0.post1 pylibcugraph-cu11-22.12.0 pylibraft-cu11-22.12.0 raft-dask-cu11-22.12.0.post1 rmm-cu11-22.12.0 treelite-3.0.1 treelite-runtime-3.0.1 ucx-py-cu11-0.29.0\nLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\nCollecting cupy-cuda11x\n  Downloading cupy_cuda11x-11.4.0-cp38-cp38-manylinux1_x86_64.whl (93.7 MB)\n     \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 93.7/93.7 MB 10.7 MB/s eta 0:00:00\nRequirement already satisfied: numpy&lt;1.26,&gt;=1.20 in /usr/local/lib/python3.8/dist-packages (from cupy-cuda11x) (1.21.6)\nRequirement already satisfied: fastrlock&gt;=0.5 in /usr/local/lib/python3.8/dist-packages (from cupy-cuda11x) (0.8.1)\nInstalling collected packages: cupy-cuda11x\nSuccessfully installed cupy-cuda11x-11.4.0\n\n          ***********************************************************************\n          With the new pip install complete, please do not run any further installation \n          commands from the conda based installation methods!!!\n\n          In your personal files, you can delete these cells.\n\n          RAPIDSAI owned templates/notebooks should already be updated with no action needed.\n          ***********************************************************************\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/#critical-imports","title":"Critical Imports","text":"<pre><code># Critical imports\nimport cudf\nimport cuml\nimport os\nimport numpy as np\nimport pandas as pd\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/#creating","title":"Creating","text":""},{"location":"machine_learning/gpu/rapids_cudf/#create-a-series-of-integers","title":"Create a Series of integers","text":"<pre><code>gdf = cudf.Series([1, 2, 3, 4, 5, 6])\nprint(gdf)\nprint(type(gdf))\n</code></pre> <pre><code>0    1\n1    2\n2    3\n3    4\n4    5\n5    6\ndtype: int64\n&lt;class 'cudf.core.series.Series'&gt;\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/#create-a-series-of-floats","title":"Create a Series of floats","text":"<pre><code>gdf = cudf.Series([1., 2., 3., 4., 5., 6.])\nprint(gdf)\n</code></pre> <pre><code>0    1.0\n1    2.0\n2    3.0\n3    4.0\n4    5.0\n5    6.0\ndtype: float64\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/#create-a-series-of-strings","title":"Create a  Series of strings","text":"<pre><code>gdf = cudf.Series(['a', 'b', 'c'])\nprint(gdf)\n</code></pre> <pre><code>0    a\n1    b\n2    c\ndtype: object\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/#create-3-column-dataframe","title":"Create 3 column DataFrame","text":"<ul> <li>Consisting of dates, integers and floats</li> </ul> <pre><code># Import\nimport datetime as dt\n\n# Using a dictionary of key-value pairs\n# Each key in the dictionary represents a category\n# The key is the category's name\n# The value is a list of the values in that category\ngdf = cudf.DataFrame({\n    # Create 10 busindates ess from 1st January 2019 via pandas\n    'dates': pd.date_range('1/1/2019', periods=10, freq='B'),\n    # Integers\n    'integers': [i for i in range(10)],\n    # Floats\n    'floats': [float(i) for i in range(10)]\n})\n\n# Print dataframe\nprint(gdf)\n</code></pre> <pre><code>       dates  integers  floats\n0 2019-01-01         0     0.0\n1 2019-01-02         1     1.0\n2 2019-01-03         2     2.0\n3 2019-01-04         3     3.0\n4 2019-01-07         4     4.0\n5 2019-01-08         5     5.0\n6 2019-01-09         6     6.0\n7 2019-01-10         7     7.0\n8 2019-01-11         8     8.0\n9 2019-01-14         9     9.0\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/#create-2-column-dataframe","title":"Create 2 column Dataframe","text":"<ul> <li>Consisting of integers and string category</li> </ul> <pre><code># Using a dictionary\n# Each key in the dictionary represents a category\n# The key is the category's name\n# The value is a list of the values in that category\ngdf = cudf.DataFrame({\n    'integers': [1 ,2, 3, 4],\n    'string': ['a', 'b', 'c', 'd']\n})\n\nprint(gdf)\n</code></pre> <pre><code>   integers string\n0         1      a\n1         2      b\n2         3      c\n3         4      d\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/#create-a-2-column-dataframe-with-pandas-bridge","title":"Create a 2 Column  Dataframe with Pandas Bridge","text":"<ul> <li>Consisting of integers and string category</li> <li>For all string columns, you must convert them to type <code>category</code> for filtering functions to work intuitively (for now)</li> </ul> <pre><code># Create pandas dataframe\npandas_df = pd.DataFrame({\n    'integers': [1, 2, 3, 4], \n    'strings': ['a', 'b', 'c', 'd']\n})\n\n# Convert string column to category format\npandas_df['strings'] = pandas_df['strings'].astype('category')\n\n# Bridge from pandas to cudf\ngdf = cudf.DataFrame.from_pandas(pandas_df)\n\n# Print dataframe\nprint(gdf)\n</code></pre> <pre><code>   integers strings\n0         1       a\n1         2       b\n2         3       c\n3         4       d\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/#viewing","title":"Viewing","text":""},{"location":"machine_learning/gpu/rapids_cudf/#printing-column-names","title":"Printing Column Names","text":"<pre><code>gdf.columns\n</code></pre> <pre><code>Index(['integers', 'strings'], dtype='object')\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/#viewing-top-of-dataframe","title":"Viewing Top of DataFrame","text":"<pre><code>num_of_rows_to_view = 2 \nprint(gdf.head(num_of_rows_to_view))\n</code></pre> <pre><code>   integers strings\n0         1       a\n1         2       b\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/#viewing-bottom-of-dataframe","title":"Viewing Bottom of DataFrame","text":"<pre><code>num_of_rows_to_view = 3 \nprint(gdf.tail(num_of_rows_to_view))\n</code></pre> <pre><code>   integers strings\n1         2       b\n2         3       c\n3         4       d\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/#filtering","title":"Filtering","text":""},{"location":"machine_learning/gpu/rapids_cudf/#method-1-query","title":"Method 1: Query","text":""},{"location":"machine_learning/gpu/rapids_cudf/#filtering-integersfloats-by-column-values","title":"Filtering Integers/Floats by Column Values","text":"<ul> <li>This only works for floats and integers, not for strings</li> </ul> <pre><code># DO NOT RUN\n# TOFIX: `cffi` package version mismatch error\nprint(gdf.query('integers == 1'))\n</code></pre> <pre><code>   integers strings\n0         1       a\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/#filtering-strings-by-column-values","title":"Filtering Strings by Column Values","text":"<ul> <li>This only works for floats and integers, not for strings so this will return an error!</li> </ul> <pre><code>print(gdf.query('strings == a'))\n</code></pre> <pre><code>---------------------------------------------------------------------------\n\nKeyError                                  Traceback (most recent call last)\n\n/usr/local/lib/python3.8/dist-packages/cudf/core/dataframe.py in extract_col(df, col)\n   7558     try:\n-&gt; 7559         return df._data[col]\n   7560     except KeyError:\n\n\n/usr/local/lib/python3.8/dist-packages/cudf/core/column_accessor.py in __getitem__(self, key)\n    154     def __getitem__(self, key: Any) -&gt; ColumnBase:\n--&gt; 155         return self._data[key]\n    156\n\n\nKeyError: 'a'\n\n\nDuring handling of the above exception, another exception occurred:\n\n\nKeyError                                  Traceback (most recent call last)\n\n&lt;ipython-input-17-5cfd0345d51c&gt; in &lt;module&gt;\n----&gt; 1 print(gdf.query('strings == a'))\n\n\n/usr/local/lib/python3.8/dist-packages/cudf/core/dataframe.py in query(self, expr, local_dict)\n   4172             }\n   4173             # Run query\n-&gt; 4174             boolmask = queryutils.query_execute(self, expr, callenv)\n   4175             return self._apply_boolean_mask(boolmask)\n   4176\n\n\n/usr/local/lib/python3.8/dist-packages/cudf/utils/queryutils.py in query_execute(df, expr, callenv)\n    212 \n    213     # prepare col args\n--&gt; 214     colarrays = [cudf.core.dataframe.extract_col(df, col) for col in columns]\n    215 \n    216     # wait to check the types until we know which cols are used\n\n\n/usr/local/lib/python3.8/dist-packages/cudf/utils/queryutils.py in &lt;listcomp&gt;(.0)\n    212 \n    213     # prepare col args\n--&gt; 214     colarrays = [cudf.core.dataframe.extract_col(df, col) for col in columns]\n    215 \n    216     # wait to check the types until we know which cols are used\n\n\n/usr/local/lib/python3.8/dist-packages/cudf/core/dataframe.py in extract_col(df, col)\n   7565         ):\n   7566             return df.index._data.columns[0]\n-&gt; 7567         return df.index._data[col]\n   7568 \n   7569\n\n\n/usr/local/lib/python3.8/dist-packages/cudf/core/column_accessor.py in __getitem__(self, key)\n    153 \n    154     def __getitem__(self, key: Any) -&gt; ColumnBase:\n--&gt; 155         return self._data[key]\n    156 \n    157     def __setitem__(self, key: Any, value: Any):\n\n\nKeyError: 'a'\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/#method-2-simple-columns","title":"Method 2:  Simple Columns","text":""},{"location":"machine_learning/gpu/rapids_cudf/#filtering-strings-by-column-values_1","title":"Filtering Strings by Column Values","text":"<pre><code># Filtering based on the string column\nprint(gdf[gdf.strings == 'b'])\n</code></pre> <pre><code>   integers strings\n1         2       b\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/#filtering-integersfloats-by-column-values_1","title":"Filtering Integers/Floats by Column Values","text":"<pre><code># Filtering based on the string column\nprint(gdf[gdf.integers == 2])\n</code></pre> <pre><code>   integers strings\n1         2       b\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/#method-2-simple-rows","title":"Method 2:  Simple Rows","text":""},{"location":"machine_learning/gpu/rapids_cudf/#filtering-by-row-numbers","title":"Filtering by Row Numbers","text":"<pre><code># Filter rows 0 to 2 (not inclusive of the third row with the index 2)\nprint(gdf[0:2])\n</code></pre> <pre><code>   integers strings\n0         1       a\n1         2       b\n</code></pre>"},{"location":"machine_learning/gpu/rapids_cudf/#method-3-locrows-columns","title":"Method 3:  loc[rows, columns]","text":"<pre><code># The syntax is as follows loc[rows, columns] allowing you to choose rows and columns accordingly\n# The example allows us to filter the first 3 rows (inclusive) of the column integers\nprint(gdf.loc[0:2, ['integers']])\n</code></pre> <pre><code>   integers\n0         1\n1         2\n2         3\n</code></pre> <pre><code>\n</code></pre>"},{"location":"news/ammi_facebook_google_recap_2018_11_21/","title":"AMMI (AIMS) Recap Supported by Facebook and Google","text":""},{"location":"news/ammi_facebook_google_recap_2018_11_21/#to-the-future-of-ai-in-and-beyond-africa","title":"To the future of AI in and beyond Africa","text":"<p>Helped to teach these students from The African Masters of Machine Intelligence (AMMI) by The African Institute for Mathematical Sciences (AIMS) with Alfredo Canziani.</p> <p>A unique pedagogy is required for deep learning beginning with intuition, culminating in just enough math, and ending with programming. This approached has worked when I delivered my deep learning course to over 3000 students over 120 countries. And I continue to champion this pedagogy. I\u2019m very pleased that I share the same passion with Alfedo when he delivered his interactive lectures.</p> <p>Importantly, I see great potential when I personally interacted with students here. I see future deep learning researchers and applied machine intelligence experts who have the potential to make groundbreaking changes in the fields of healthcare, agriculture and education throughout Africa. With the right education, mentorship and opportunities, they would be pioneers. Simply put, deep learning talent is underexplored in this continent.</p> <p>I am thankful to Alfredo Canziani who has been my best pal and inspirational guy while I was there, Moustapha Cisse for creating these opportunities for young African students to thrive, #Facebook and #Google for supporting this program, and Pedro Antonio Mart\u00ednez Mediano and Marc Deisenroff. Coincidentally, I would also like to thank Yoshua Bengio for holding ICLR in Africa in 2020 for the first time that he announced recently. Finally, really appreciate how #PyTorch has made programming neural networks more approachable and it was made possible with the amazing PyTorch community sparked by Soumith Chintala.</p> <p>Speaking about community, I got to contribute to the deep learning community with the help from supportive groups and individuals from the global deep learning community. Likewise, we should continually help these young Africans like any other.</p> <p>To the future of AI in and beyond Africa. Looking forward to help Moustapha in any way I can to nurture AI talent \ud83d\ude42</p> <p>I'm being a fanboy here, but Yann Lecun liked my post on Facebook! </p> <p>Cheers, Ritchie</p>"},{"location":"news/dbs_gpu_rapids_nvidia_ensemble_frac_diff/","title":"GPU Fractional Differencing with NVIDIA RAPIDS","text":""},{"location":"news/dbs_gpu_rapids_nvidia_ensemble_frac_diff/#introduction-to-deep-learning","title":"Introduction to Deep Learning","text":"<p>I created and led a workshop on accelerating fractional differencing with NVIDIA's RAPIDS in DBS.</p> <p>This is our article on medium with NVIDIA on our work.</p> <p>You can check out our presentation and open-source code on Github.</p> <p></p> <p>Thanks to our industry partners DBS, NVIDIA and ensemblecap.ai</p> <p>Cheers, Ritchie</p>"},{"location":"news/deep_learning_wizard_1y_2018_06_01/","title":"Featured on PyTorch Website","text":""},{"location":"news/deep_learning_wizard_1y_2018_06_01/#pytorch-a-year-later","title":"PyTorch a Year Later","text":"<p>We are featured on PyTorch website's post </p> <p>I used PyTorch from day 1 and I fell in love with it. One year later, PyTorch's community has grown tremendously. I thank Facebook AI Research (FAIR) and everyone who contributed to building one of the best deep learning framework. In particular Soumith Chintala from FAIR where he is continually fixing bugs and making PyTorch better everyday.</p> <p>A year later, Deep Learning Wizard has taught more than 2000 students across 100+ countries, and we continue to strive to release new and updated content to enable anyone to build customized deep learning systems to solve many problems. And I'm trying to push for PyTorch workshops at NVIDIA Deep Learning Institute in Singapore and across the world, so stay tuned!</p> <p>A big shoutout for Alfredo Canziani who's currently a postdoc at NYU co-teaching with Yann LeCun on deep learning classes and he's releasing another technical course on PyTorch that is going to be awesome.</p> <p>To more great years ahead for PyTorch </p> <p>Cheers, Ritchie Ng</p>"},{"location":"news/deep_learning_wizard_nvidia_inception_2018_05_01/","title":"We Are an NVIDIA Inception Partner","text":""},{"location":"news/deep_learning_wizard_nvidia_inception_2018_05_01/#we-did-it","title":"We did it!","text":"<p>After almost a year, we are an NVIDIA Inception Partner now! </p> <p>\"NVIDIA Inception Partner</p> <p></p> <p>Deep Learning Wizard is proudly an NVIDIA Inception Partner, empowering people across the world to leverage on deep learning via open-source programming languages and frameworks to solve problems. </p> <p>Cheers, Ritchie Ng</p>"},{"location":"news/defence_and_science_technology_agency_dsta_nvidia_talk_2016_06/","title":"Deep Learning Demystified, Defence and Science Technology Agency (DSTA)","text":""},{"location":"news/defence_and_science_technology_agency_dsta_nvidia_talk_2016_06/#introduction-to-deep-learning","title":"Introduction to Deep Learning","text":"<p>I gave an introductory talk on deep learning to more than 1000 students at BrainHack held by DSTA in June 2019.</p> <p>I'm really excited for Singapore's progress in leveraging on deep learning for the defense sector. There are a lot of applications and areas of research.</p> <p> </p> <p>Special thanks to DSTA, NVIDIA and ensemblecap.ai</p> <p>Cheers, Ritchie</p>"},{"location":"news/detect_waterbone_debris_ai_for_social_good_icml_2019_06/","title":"Oral Presentation for AI for Social Good Workshop ICML","text":""},{"location":"news/detect_waterbone_debris_ai_for_social_good_icml_2019_06/#detecting-waterborne-debris-with-sim2real-and-randomization","title":"Detecting Waterborne Debris with Sim2Real and Randomization","text":"<p>We'll be presenting our work in ICML this June.</p> <p>Special thanks to everyone working on this project: </p> <p>Jie Fu (MILA, Polytechnic Montr\u00e9al); Ritchie Ng (National University of Singapore); Mirgahney Mohamed (The African Institute for Mathematical Sciences); Yi Tay (Nanyang Technological University); Kris Sankaran (Montreal Institute for Learning Algorithms); Shangbang Long (Peking University); Alfredo Canziani (New York University); Chris J Pal (MILA, Polytechnique Montr\u00e9al, Element AI); Moustapha Cisse (Google Research)</p> <p>Cheers, Ritchie</p>"},{"location":"news/facebook_pytorch_devcon_recap_2018_10_02/","title":"Recap of Facebook PyTorch Devcon","text":""},{"location":"news/facebook_pytorch_devcon_recap_2018_10_02/#woo","title":"Woo!","text":"<p>Proud of how far PyTorch has come. Thanks Soumith Chintala, it is a great journey and it just started. And thanks to Adam Paszke too!</p> <p>Finally got to catch up with Soumith Chintala, Alfredo Canziani and Marek Bardo\u0144ski and found Andrej Karpathy! Back to Singapore tonight \ud83e\udd17</p> <p>Here are some photos of my trip! </p> <p> </p> <p>Cheers, Ritchie</p>"},{"location":"news/facebook_pytorch_developer_conference_2018_09_05/","title":"Facebook PyTorch Developer Conference","text":""},{"location":"news/facebook_pytorch_developer_conference_2018_09_05/#we-are-heading-down","title":"We are heading down!","text":"<p>In barely 2 short years, PyTorch (Facebook) will be hosting their first PyTorch Developer Conference in San Francisco, USA.</p> <p>I will be heading down thanks to Soumith Chintala for the invite and arrangements. Looking forward to meet anyone there. </p> <p>The PyTorch ecosystem has grown tremendously from when I first started using it. To this date, I've taught more than 3000 students worldwide in 120+ countries and every single wizard has fallen in love with it!</p> <p>Cheers, Ritchie Ng</p>"},{"location":"news/it_youth_leader_2019_03_11/","title":"IT Youth Leader of the Year 2019 Award","text":""},{"location":"news/it_youth_leader_2019_03_11/#we-got-it","title":"We got it!","text":"<p>I am very happy to be the IT Youth Leader of the Year 2019 in Singapore for my industry and non-profit contribution in AI at Deep Learning Wizard, ensemblecap.ai, NUS,  and NVIDIA!</p> <p>I would like to thank the professor who changed my life, believed in me and nominated me for consideration, Professor Chua Tat-Seng from NExT Centre, NUS School of Computing. Big thanks to everyone who helped for this in NExT Centre particularly Choon Meng, Tek Min, Chua Yi Hao, and Eugene Tan.</p> <p>Special thanks to an important friend and associate who stuck by with me all the way, Jie Fu from our endless days of programming (and scribbling equations) during your PhD years to now as a postdoc in MILA \ud83d\ude42And Ilija Ilievski from those years till now too!</p> <p>Big thanks to the people who believed in the future of AI and asset management and made this happen with the original team Damien Loh, Atsuo Ogaki, and Samuel Chen from ensemblecap.ai</p> <p>Thanks to new found close friend Alfredo Canziani who motivated me on open-source education further and supporting each other \ud83d\ude42 And thanks to Moustapha Cisse for setting up an amazing initiative in Africa with #AMMI to kickstart a future of AI by Africa and trusted your students with our teaching. And I'm on my starting journey to help as many students of your as possible by guiding them on learning like Merghaney Mohammed and Enoch Tetteh.</p> <p>Thanks to Soumith Chintala for spearheading #PyTorch and it's now in production running critical functions in finance (like my firm) and tech.</p> <p>Thanks to Marek Bardo\u0144ski where our friendship started when you were in #NVIDIA and we've supported each other ever since in our learning journey.</p> <p>Thanks to #NVIDIA for allowing me to lead deep learning workshops under DLI in NUS all these years.</p> <p>Thanks to my mentors and everyone in #PhilipYeoInitiative like Abel Ang, Seok Lin Khoo, Kiren Kumar, Claire Cheong, Chong Siak Ching, and Kai Hoe Tan</p> <p>Last thanks to everyone in #SingaporeComputerSociety like Jason Chen for being there when I was nervous and Howie Lau for spearheading tech from start till now. And to a senior big award winner I've a lot to learn and also from NUS: Jeffrey Tiong!</p> <p>Thanks to forward-looking civil servants and minister like Minister S Iswaran for gracing the event and delivering a speech on digital readiness and skills empowerment for Singaporeans.</p> <p>Thanks to my family for being by my side Carine Chng, Ng William, and Adabelle Ng for giving me the freedom and support to pursue whatever I wanted even from young.</p> <p>To the future of AI!</p> <p>Cheers, Ritchie</p>"},{"location":"news/nanjing_next_nus_tsinghua_ai_finance_healthcare_2018_11_01/","title":"NExT++ AI in Finance and Healthcare Workshop 2018","text":""},{"location":"news/nanjing_next_nus_tsinghua_ai_finance_healthcare_2018_11_01/#recap","title":"Recap","text":"<p>This is NExT++\u2019s 3<sup>rd</sup> Workshop and this time we are looking at the applications and development of AI related technologies in the healthcare and finance verticals with the theme \u201cAI in Health and Finance\u201d</p> <p>Visual summary of my talk on AI and Unstructured Analytics in Fintech. Doesn't contain everything, but whatever the non-technical designers could come up with in real-time. Really amazing so kudos to them.</p> <p>Really thankful to the whole team in my former lab, NExT++ and a close friend, Choon Meng, for making all the arrangements. Also thanks to Tek Min, Yi Hao and everyone else.</p> <p>Super happy to catch up with Prof Tat-Seng Chua (KITHCT Chair Professor at the School of Computing), Prof Sun Maosong (Dean of Department of Computer Science and Technology, Tsinghua University), and Prof Dame Wendy Hall (Director of the Web Science Institute, University of Southampton).</p> <p></p> <p>Cheers, Ritchie</p>"},{"location":"news/news/","title":"Welcome to our Blog","text":"<p>Here, we post news related to Deep Learning Wizard's releases, features and achievements </p>"},{"location":"news/news/#notable-news","title":"Notable News","text":"<ul> <li> Oral Presentation for AI for Social Good Workshop ICML, Long Beach, Los Angeles, USA, June 2019</li> <li> IT Youth Leader of The Year, Singapore Computer Society (SCS), Singapore, March 2019</li> <li> Foundations of Deep Learning, African Masters of Machine Intelligence (AMMI), Google &amp; Facebook, Kigali, Rwanda, November 2018</li> <li> Invited to Facebook PyTorch Developer Conference, San Francisco, USA, September 2018</li> <li> Led NUS-MIT-NUHS Datathon NVIDIA Image Recognition Workshop, Singapore, July 2018</li> <li> Featured on PyTorch Website, January 2018</li> <li> Led NVIDIA Self-Driving Cars and Healthcare Talk, Singapore, June 2017</li> <li> Clinched NVIDIA Inception Partner Status, Singapore, May 2017</li> </ul>"},{"location":"news/nvidia_nus_mit_datathon_2018_07_05/","title":"NVIDIA Workshop at NUS-MIT-NUHS Datathon","text":""},{"location":"news/nvidia_nus_mit_datathon_2018_07_05/#image-recognition-workshop-by-ritchie-ng","title":"Image Recognition Workshop by Ritchie Ng","text":"<p>The NVIDIA Deep Learning Institute (DLI) trains developers, data scientists, and researchers on how to use artificial intelligence to solve real-world problems across a wide range of domains. In the deep learning courses, you\u2019ll learn how to train, optimize, and deploy neural networks using the latest tools, frameworks, and techniques for deep learning.</p> <p>In \u201cImage Classification with DIGITS\u201d mini-course, you will learn how to train a deep neural network to recognize handwritten digits by loading image data into a training environment, choosing and training a network, testing with new data, and iterating to improve performance.</p> <p>Link to the NUS-MIT-NUHS Datathon workshop.</p>"},{"location":"news/nvidia_self_driving_cars_talk_2017_06_21/","title":"NVIDIA Self-Driving Cars and Healthcare Workshop","text":""},{"location":"news/nvidia_self_driving_cars_talk_2017_06_21/#hosted-by-ritchie-ng","title":"Hosted by Ritchie Ng","text":"<p>A talk by Marek Bardo\u0144ski, Senior Deep Learning Research Engineer, NVIDIA Switzerland. This is hosted by Ritchie Ng, Deep Learning Researcher, NExT Search Centre, NUS.</p> <p>We will be touching on cutting-edge applications of deep learning for self-driving cars and medical diagnostics. Also there will be a tutorial followed by networking with deep learning researchers from NUS and NVIDIA.</p> <p>Details: Wednesday, June 21<sup>st</sup> 1:00 PM to 3:30 PM</p> <p>The Hangar by NUS Enterprise 21 Heng Mui Keng Terrace, Singapore 119613</p> <p></p>"},{"location":"ood/intro/","title":"Out-of-distribution Data","text":"<p>This is a persistent and critical production issue present in any machine learning and deep learning systems, no matter how good the models were trained.</p> <p>A single out-of-distribution sample data fed into a well-trained model can result in a few problematic outcomes, and in production-level systems they become fat-tailed critical risks that have outsized consequences.</p> Problematic consequences of out-of-distribution data <ol> <li>False positive &amp; False Negative: prediction does not match the ground truth.</li> <li>Unreliable True positive &amp; True Negative: prediction matches the ground truth but due to samples being out-of-distribution, the performance of these predictions becomes very erratic. </li> </ol>"},{"location":"programming/intro/","title":"Programming","text":"<p>We'll be covering C++, Python, Bash and more for end-to-end AI deployments.</p> Packages and Languages you will learn to use <ul> <li>C++</li> <li>Python</li> <li>Bash</li> <li>Matplotlib</li> <li>Javascript</li> <li>Electron</li> </ul> <p>Work in progress</p> <p>This open-source portion is still a work in progress. Stay tuned while we gradually upload our tutorials and notes. Feel free to contact Ritchie Ng if you would like to contribute via our Facebook page.</p>"},{"location":"programming/bash/bash/","title":"Bash","text":"<p>Run Bash Files</p> <p>You can find the bash code files for this section in this link.</p>"},{"location":"programming/bash/bash/#creating-and-running-a-bash-file","title":"Creating and Running a Bash File","text":""},{"location":"programming/bash/bash/#create-bash-file","title":"Create bash file","text":"<pre><code>touch hello_world.sh\n</code></pre>"},{"location":"programming/bash/bash/#edit-bash-file-with-hello-world","title":"Edit bash file with Hello World","text":"<p>You can edit with anything like Vim, Sublime, etc.</p> <pre><code>echo Hello World!\n</code></pre>"},{"location":"programming/bash/bash/#run-bash-file","title":"Run bash file","text":"<pre><code>bash hello_world.sh\n</code></pre> <p>This will print out, in your bash: <pre><code>Hello World!\n</code></pre></p>"},{"location":"programming/bash/bash/#calculating","title":"Calculating","text":"<pre><code># Add\necho $((10+10))\n\n# Subtract\necho $((10-10))\n\n# Divide\necho $((10/10))\n\n# Multiple\necho $((10*10))\n\n# Modulo\necho $((10%10))\n\n# Multiple Operations: Divide and Add\necho $((10/10 + 10))\n</code></pre> <pre><code>20\n0\n1\n100\n0\n11\n</code></pre>"},{"location":"programming/bash/bash/#loops-and-conditional","title":"Loops and Conditional","text":""},{"location":"programming/bash/bash/#for-loop","title":"For Loop","text":"<pre><code>for i in 'A' 'B' 'C'\n    do\n        echo $i\n    done\n</code></pre> <pre><code>A\nB\nC\n</code></pre>"},{"location":"programming/bash/bash/#for-loop-with-range","title":"For Loop With Range","text":"<p>This will echo the digits 0 to 10 without explicitly requiring to define the whole range of numbers/alphabets like above.</p> <pre><code>for ((i=0; i&lt;=10; i++));\n    do\n        echo $i\n    done\n</code></pre> <pre><code>0\n1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n</code></pre>"},{"location":"programming/bash/bash/#if-else-conditional","title":"If Else Conditional","text":"<p>This is a simple if-else to check if the day of the week is 5, meaning if it is a Friday.</p> <pre><code>day=$(date +%u)\n\nif [ $day == 5 ];\n    then\n        echo \"Friday is here!\"\n\n    else\n        echo \"Friday is not here :(\"\n        echo \"Today is day $day of the week\"\n    fi\n</code></pre>"},{"location":"programming/bash/bash/#sequentially-running-of-python-scripts","title":"Sequentially Running of Python Scripts","text":"<p>This snippet allows you to to run 3 python scripts sequentially, waiting for each to finish before proceeding to the next.</p> <pre><code>python script_1.py\nwait\n\npython script_2.py \nwait\n\npython script_3.py\nwait\n\necho \"Finished running all 3 scripts in sequence!\"\n</code></pre>"},{"location":"programming/bash/bash/#parallel-running-of-python-scripts","title":"Parallel Running of Python Scripts","text":"<pre><code>python script_1.py &amp;&amp; script_2.py &amp;&amp; script_3.py\nwait\n\necho \"Finished running all 3 scripts in parallel in sequence\"\n</code></pre>"},{"location":"programming/bash/bash/#reading-and-writing-operations","title":"Reading and Writing Operations","text":""},{"location":"programming/bash/bash/#reading-logs-and-texts","title":"Reading logs and texts","text":"<p>Create a text file called <code>random_text.txt</code> with the following contents</p> <pre><code>Row 1\nRow 2\nRow 3\nRow 4\nRow 5\nRow 6\nRow 7\nRow 8\nRow 9\nRow 10\n</code></pre> <p>Then run the following command to read it in bash then print it.</p> <pre><code>text_file=$(cat random_text.txt)\necho $text_file\n</code></pre> <pre><code>Row 1 Row 2 Row 3 Row 4 Row 5 Row 6 Row 7 Row 8 Row 9 Row 10\n</code></pre>"},{"location":"programming/bash/bash/#date-operations","title":"Date Operations","text":""},{"location":"programming/bash/bash/#getting-current-date","title":"Getting Current Date","text":"<p>This will return the date in the format YYYY-MM-DD for example 2019-06-03.</p> <pre><code>DATE=`date +%Y-%m-%d`\necho $DATE\n</code></pre>"},{"location":"programming/bash/bash/#getting-current-day-of-week","title":"Getting Current Day of Week","text":"<p>This will return 1, 2, 3, 4, 5, 6, 7 depending on the day of the week.</p> <pre><code>DAY=$(date +%u)\necho $DAY\n</code></pre>"},{"location":"programming/bash/bash/#changing-system-dates-by-1-day","title":"Changing System Dates By 1 Day","text":"<p>You can change system dates based on this. Surprisingly, you'll find it useful for testing an environment for deployments in the next day and then shifting it back to the actual day.</p> <pre><code>sudo date -s 'next day'\nsudo date -s 'yesterday'\n</code></pre> <p>If you are running some tests via bash and want to disable typing in password you can edit the sudoer file via <code>sudo visudo</code> and adding the following line. Only use <code>sudo visudo</code> and nothing else, as they've a special syntax.</p> <pre><code>&lt;username&gt; ALL=(ALL) NOPASSWD: /bin/date\n</code></pre> <p>To find out your username, simply just run the command <code>whoami</code>.</p>"},{"location":"programming/bash/bash/#jupyter-utility-commands","title":"Jupyter Utility Commands","text":""},{"location":"programming/bash/bash/#convert-notebook-to-htmlmarkdown","title":"Convert Notebook to HTML/Markdown","text":"<pre><code>jupyter nbconvert --to markdown python.ipynb\njupyter nbconvert --to html python.ipynb\n</code></pre>"},{"location":"programming/bash/bash/#bash-convenient-commands","title":"Bash Convenient Commands","text":""},{"location":"programming/bash/bash/#list-directories-only","title":"List directories only","text":"<p><code>ls -d */</code></p>"},{"location":"programming/bash/bash/#list-non-directories-only","title":"List non-directories only","text":"<p><code>ls -p | grep -v '/$'</code></p>"},{"location":"programming/bash/bash/#check-ip","title":"Check IP","text":"<p><code>ifconfig | sed -En \"s/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\\.){3}[0-9]*).*/\\2/p\"</code></p>"},{"location":"programming/bash/bash/#check-internet-speed","title":"Check internet speed","text":"<p><code>ethtool eno1</code></p>"},{"location":"programming/bash/bash/#check-disk-space","title":"Check disk space","text":"<p><code>df -h</code></p>"},{"location":"programming/bash/bash/#check-ubuntu-version","title":"Check ubuntu version","text":"<p><code>lsb_release -a</code></p>"},{"location":"programming/bash/bash/#check-truncated-system-logs","title":"Check truncated system logs","text":"<p><code>tail /var/log/syslog</code></p>"},{"location":"programming/bash/bash/#check-cuda-version","title":"Check CUDA version","text":"<p><code>nvcc -V</code></p>"},{"location":"programming/bash/bash/#check-cudnn-version","title":"Check cuDNN version","text":"<p><code>cat /usr/local/cuda/include/cudnn.h | grep CUDNN_MAJOR -A 2</code></p>"},{"location":"programming/bash/bash/#check-username","title":"Check username","text":"<p><code>whoami</code></p>"},{"location":"programming/bash/bash/#untar-file","title":"Untar file","text":"<p><code>tar -xvzf file_name</code></p>"},{"location":"programming/bash/bash/#open-pdf-file","title":"Open PDF file","text":"<p><code>gvfs-open file_name</code></p>"},{"location":"programming/bash/bash/#download-file-from-link-rapidly-with-aria","title":"Download file from link rapidly with aria","text":"<p><code>aria2c -x16 -c url_link</code></p>"},{"location":"programming/bash/bash/#kill-all-python-processes","title":"Kill all python processes","text":"<p><code>ps ax | grep python | cut -c1-5 | xargs kill -9</code></p>"},{"location":"programming/bash/bash/#install-deb-files","title":"Install .deb files","text":"<p><code>sudo apt-get install -f file_name.deb</code></p>"},{"location":"programming/bash/bash/#empty-thrash","title":"Empty thrash","text":"<pre><code>sudo apt-get install trash-cli\nthrash-empty\n</code></pre>"},{"location":"programming/bash/bash/#fix-git-permissions","title":"Fix Git Permissions","text":"<p>There are times whether you're on Windows (frequent), Mac, or Linux, you might encounter a permission error although you have obviously set it up correctly and it was last working. This is quick fix you can run. </p> <pre><code>eval `ssh-agent -s` &amp;&amp; ssh-add ~/.ssh/github\n</code></pre> <p>If you want, you can even make an alias <code>fixgit</code> and quickly call it in bash to make it easier to fix whenever you face this issue. It's a quick fix, there're more permnanent ways to fix it if you want. But this is a quick fix section. </p> <p>In your <code>.bashrc</code> or <code>.zshrc</code>, include the following:</p> <pre><code>alias fixgit='eval `ssh-agent -s` &amp;&amp; ssh-add ~/.ssh/github'\n</code></pre>"},{"location":"programming/bash/bash/#get-git-repo-information","title":"Get Git Repo Information","text":"<pre><code>curl https://api.github.com/&lt;user&gt;/&lt;repo_name&gt;\n</code></pre>"},{"location":"programming/bash/bash/#conda-commands","title":"Conda Commands","text":""},{"location":"programming/bash/bash/#check-conda-environment","title":"Check conda environment","text":"<p><code>conda env list</code></p>"},{"location":"programming/bash/bash/#create-conda-kernel","title":"Create conda kernel","text":"<pre><code>conda create -n kernel_name python=3.6\nsource activate kernel_name\n</code></pre>"},{"location":"programming/bash/bash/#install-conda-kernel","title":"Install conda kernel","text":"<pre><code>conda install ipykernel\nsource activate kernel_name\npython -m ipykernel install --user --name kernel_name --display-name kernel_name\n</code></pre>"},{"location":"programming/bash/bash/#remove-conda-kernel","title":"Remove conda kernel","text":"<p><code>conda env remove -n kernel_name</code></p>"},{"location":"programming/bash/bash/#recovering-problematic-conda-installation","title":"Recovering problematic conda installation","text":"<pre><code># Download miniconda according to your environment\n# Link: https://docs.conda.io/en/latest/miniconda.html\n\n# Backup existing miniconda environment that may have problems\nmv miniconda3 miniconda3_backup\n\n# Install miniconda\nbash Miniconda3-latest-Linux-x86_64.sh\n\n# Restore old environment settings\nrsync -a miniconda3_backup/ miniconda3/\n</code></pre>"},{"location":"programming/bash/bash/#internet-operations","title":"Internet Operations","text":""},{"location":"programming/bash/bash/#checking-internet-availability","title":"Checking Internet Availability","text":"<p>This script will return whether your internet is fine or not without using pings.</p> <p>Pings can often be rendered unusable when the network administrator disables ICMP to prevent the origination of ping floods from the data centre.</p> <pre><code>if nc -zw1 google.com 443;\n    then\n        echo \"INTERNET: OK\"\n    else\n        echo \"INTERNET: NOT OK\"\n    fi\n</code></pre>"},{"location":"programming/bash/bash/#cron-operations","title":"Cron Operations","text":""},{"location":"programming/bash/bash/#edit-cron","title":"Edit Cron","text":"<p>Formatting follows this syntax with full credits on this beautiful diagram to fedorqui from Stack Overflow: <pre><code> \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 minute (0 - 59)\n \u2502 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 hour (0 - 23)\n \u2502 \u2502 \u250c\u2500\u2500\u2500\u2500\u2500\u2500 day of month (1 - 31)\n \u2502 \u2502 \u2502 \u250c\u2500\u2500\u2500\u2500 month (1 - 12)\n \u2502 \u2502 \u2502 \u2502 \u250c\u2500\u2500 day of week (0 - 6 =&gt; Sunday - Saturday, or\n \u2502 \u2502 \u2502 \u2502 \u2502                1 - 7 =&gt; Monday - Sunday)\n \u2193 \u2193 \u2193 \u2193 \u2193\n * * * * * command to be executed\n</code></pre></p> <p>Edit cron with this command: <pre><code>sudo crontab -e\n</code></pre></p>"},{"location":"programming/bash/bash/#list-cron","title":"List Cron","text":"<pre><code>sudo crontab -l\n</code></pre>"},{"location":"programming/bash/bash/#status-start-stop-and-restart","title":"Status, Start, Stop and Restart","text":"<pre><code>sudo service cron status\nsudo service cron stop\nsudo service cron start\nsudo service cron restart\n</code></pre>"},{"location":"programming/bash/bash/#cron-debugging","title":"Cron Debugging","text":"<p>Install postfix for local routing of errors (choose local option): <pre><code>sudo apt-get install postfix\n</code></pre></p> <p>Restart cron to see for any errors posted (if not errors, there will be no file, be patient before errors are posted): <pre><code>sudo cat /var/mail/root\n</code></pre></p>"},{"location":"programming/bash/bash/#cron-bash-fix","title":"Cron Bash Fix","text":"<p>Cron uses <code>/bin/sh</code> as the default shell. Typically you would realize you're using <code>/bin/bash</code> as your shell, so this typically needs to be rectified before you can use cron to schedule commands as if it were your usual bash.</p> <p>Edit your cron file via <code>sudo crontab -e</code> and paste the following lines at the end of the file prior to your command like so. Take note for <code>PATH</code>, you've to paste the output of <code>echo PATH=$PATH</code> in there instead!</p> <pre><code>SHELL=/bin/bash\nPATH=/usr/lib....\n# Your command schedule here!\n</code></pre>"},{"location":"programming/bash/bash/#cron-conda-environment","title":"Cron Conda Environment","text":"<p>This is an example of enabling an anaconda environment, for example the default <code>base</code>, and running a python script. </p> <p>Take note you need to put your python script in the right directory or simply navigate to that path with <code>cd</code> prior to <code>\"$(conda shell.bash hook)\"</code>. </p> <pre><code>SHELL=/bin/bash\nPATH=/usr/lib....\n* * * * 1-5 eval \"$(conda shell.bash hook)\" &amp;&amp; conda activate base &amp;&amp; python python_script_name.py\n</code></pre>"},{"location":"programming/bash/bash/#cron-running-processes","title":"Cron Running Processes","text":"<p>Some times you want to see the status of running tasks and may want to get the PID to end it. This is a very handy command.</p> <pre><code>ps -o pid,sess,cmd afx | egrep -A20 \"( |/)cron( -f)?$\"\n</code></pre> <p>You can get the PID of the cron process and then end it with <code>sudo pkill -s &lt;PID&gt;</code></p>"},{"location":"programming/bash/bash/#hardware-information","title":"Hardware Information","text":""},{"location":"programming/bash/bash/#comprehensive-cpu-information","title":"Comprehensive CPU Information","text":"<pre><code>cat /proc/cpuinfo\n</code></pre>"},{"location":"programming/bash/bash/#number-of-cpu-threads","title":"Number of CPU Threads","text":"<pre><code>!grep -c ^processor /proc/cpuinfo\n</code></pre> <p>or </p> <pre><code>nproc\n</code></pre>"},{"location":"programming/bash/bash/#cpu-model-name","title":"CPU Model Name","text":"<pre><code>!cat /proc/cpuinfo | grep \"model name\" \n</code></pre>"},{"location":"programming/bash/bash/#check-available-ram","title":"Check Available RAM","text":""},{"location":"programming/bash/bash/#in-mb","title":"In MB","text":"<pre><code>free -m\n</code></pre>"},{"location":"programming/bash/bash/#in-gb","title":"In GB","text":"<pre><code>free -g\n</code></pre>"},{"location":"programming/cpp/cpp/","title":"C++","text":"<p>Run Jupyter Notebook</p> <p>You can run the code for this section in this jupyter notebook link.</p>"},{"location":"programming/cpp/cpp/#installation-of-interactive-c17","title":"Installation of Interactive C++17","text":"<p>Xeus-Cling is a game-changer where similar to Python Jupyter Notebooks, we can run C++ Jupyter Notebooks now. Run the following bash commands in sequence to create a C++ kernel for Jupyter Notebook.</p> <pre><code>conda create -n cpp\nsource activate cpp\nconda install -c conda-forge xeus-cling\njupyter kernelspec install --user /home/ritchie/miniconda3/envs/cpp/share/jupyter/kernels/xcpp17\njupyter notebook\n</code></pre>"},{"location":"programming/cpp/cpp/#printing","title":"Printing","text":""},{"location":"programming/cpp/cpp/#printing-single-line","title":"Printing Single Line","text":"<pre><code>// When you compile, the preprocessor runs and acts on all line with the pound key first\n\n// This is a preprocessor instruction that\n// essentially places the file with the name iostream into this spot\n#include &lt;iostream&gt;\n</code></pre> <pre><code>// Print one line\nstd::cout &lt;&lt; \"Using Xeus Cling\" &lt;&lt; std::endl;\n</code></pre> <pre><code>Using Xeus Cling\n</code></pre>"},{"location":"programming/cpp/cpp/#printing-2-lines","title":"Printing 2 Lines","text":"<pre><code>// Print two lines\nstd::cout &lt;&lt; \"First Line \\nSecond Line\" &lt;&lt; std::endl;\n</code></pre> <pre><code>First Line \nSecond Line\n</code></pre>"},{"location":"programming/cpp/cpp/#printing-with-tabs","title":"Printing with Tabs","text":"<pre><code>// Print numbers with nicely formatted tabs\nstd::cout &lt;&lt; \"One hundred:\\t\";\nstd::cout &lt;&lt; (float) 1000/10 &lt;&lt; std::endl;\n\nstd::cout &lt;&lt; \"Two hundred:\\t\";\nstd::cout &lt;&lt; (double) 2000/10 &lt;&lt; std::endl;\n\nstd::cout &lt;&lt; \"Three hundred:\\t\";\nstd::cout &lt;&lt; (double) 3000/10 &lt;&lt; std::endl;\n</code></pre> <pre><code>One hundred:    100\nTwo hundred:    200\nThree hundred:  300\n</code></pre>"},{"location":"programming/cpp/cpp/#easier-printing-subjective-with-namespaces","title":"Easier Printing (subjective) with Namespaces","text":"<ul> <li>Gets irritating to use <code>std</code> in front of <code>cout</code> and <code>endl</code> to keep printing so we can use namespaces</li> </ul> <pre><code>using namespace std;\ncout &lt;&lt; \"No need for messy std::\" &lt;&lt; endl;\n</code></pre> <pre><code>No need for messy std::\n</code></pre>"},{"location":"programming/cpp/cpp/#variables","title":"Variables","text":""},{"location":"programming/cpp/cpp/#general-memory-management","title":"General memory management","text":"<p>In C++, you always need to determine each variable's type so the compiler will know how much memory (RAM) to allocate to the variable</p>"},{"location":"programming/cpp/cpp/#types-of-variables","title":"Types of Variables","text":"Type Bytes (Size) Range of Values char 1 0 to 255 or -127 to 127 unsigned char 1 -127 to 127 signed char 1 0 to 255 bool 1 True or False int 4 -2,147,483,648 to 2,147,483,647 unsigned int 4 0 to 4,294,967,295 signed int 4 -2,147,483,648 to 2,147,483,647 short int 2 -32,768 to 32,767 long int 4 -2,147,483,648 to 2,147,483,647 float 4 3.4e-38 to 3.4e38 double 8 1.7e-308 to 1.7e-08 long double 8 1.7e-308 to 1.7e-08"},{"location":"programming/cpp/cpp/#integer-sizes","title":"Integer sizes","text":"<ul> <li>Typical sizes<ul> <li>Short integer: 2 bytes (will be smaller than long)<ul> <li>Limitation is that the value of this short integer has a max value, you should be cautious of using short integers</li> </ul> </li> <li>Long integer: 4 bytes</li> <li>Integer: 2 or 4 bytes</li> </ul> </li> <li>Notes<ul> <li>Technically these sizes can vary depending on your processor (32/64 bit) and compiler</li> <li>You should not assume the sizes but the hierarchy of sizes will not change (short memory &lt; long memory)</li> </ul> </li> </ul>"},{"location":"programming/cpp/cpp/#integer-size","title":"Integer size","text":"<pre><code>cout &lt;&lt; \"Size of an integer: \" &lt;&lt; sizeof (int);\n</code></pre> <pre><code>Size of an integer: 4\n</code></pre>"},{"location":"programming/cpp/cpp/#short-integer-size","title":"Short integer size","text":"<pre><code>cout &lt;&lt; \"Size of a short integer: \" &lt;&lt; sizeof (short int);\n</code></pre> <pre><code>Size of a short integer: 2\n</code></pre>"},{"location":"programming/cpp/cpp/#long-integer-size","title":"Long integer size","text":"<pre><code>cout &lt;&lt; \"Size of an long integer: \" &lt;&lt; sizeof (long int);\n</code></pre> <pre><code>Size of an long integer: 8\n</code></pre>"},{"location":"programming/cpp/cpp/#unsigned-or-signed-integers","title":"Unsigned or signed integers","text":"<ul> <li>Unsigned integers: only can hold positive integers</li> <li>Signed integers: can hold positive/negative integers</li> </ul>"},{"location":"programming/cpp/cpp/#signed-short-integer","title":"Signed short integer","text":"<pre><code>cout &lt;&lt; \"Size of an signed short integer: \" &lt;&lt; sizeof (signed short int);\n</code></pre> <pre><code>Size of an signed short integer: 2\n</code></pre>"},{"location":"programming/cpp/cpp/#unsigned-short-integer","title":"Unsigned short integer","text":"<pre><code>cout &lt;&lt; \"Size of an unsigned short integer: \" &lt;&lt; sizeof (unsigned short int);\n</code></pre> <pre><code>Size of an unsigned short integer: 2\n</code></pre>"},{"location":"programming/cpp/cpp/#signed-long-integer","title":"Signed long integer","text":"<pre><code>cout &lt;&lt; \"Size of an signed long integer: \" &lt;&lt; sizeof (signed long int);\n</code></pre> <pre><code>Size of an signed long integer: 8\n</code></pre>"},{"location":"programming/cpp/cpp/#unsigned-long-integer","title":"Unsigned long integer","text":"<pre><code>cout &lt;&lt; \"Size of an unsigned long integer: \" &lt;&lt; sizeof (unsigned long int);\n</code></pre> <pre><code>Size of an unsigned long integer: 8\n</code></pre>"},{"location":"programming/cpp/cpp/#constants","title":"Constants","text":""},{"location":"programming/cpp/cpp/#literal-constants","title":"Literal Constants","text":"<pre><code>int varOne = 20;\nstd::cout &lt;&lt; varOne &lt;&lt; std::endl;\n</code></pre> <pre><code>20\n</code></pre>"},{"location":"programming/cpp/cpp/#enumerated-constants","title":"Enumerated Constants","text":"<p>This enables you to create a new type! In this example we create a new type <code>directions</code> containing <code>Up</code>, <code>Down</code>, <code>Left</code>, and <code>Right</code>.</p> <pre><code>enum directions {Up, Down, Left, Right};\n\ndirections goWhere;\ngoWhere = Right;\n\nif (goWhere == Right)\n    std::cout &lt;&lt; \"Go right\" &lt;&lt; std::endl;\n</code></pre> <pre><code>Go right\n</code></pre>"},{"location":"programming/cpp/cpp/#functions","title":"Functions","text":""},{"location":"programming/cpp/cpp/#function-without-return-value","title":"Function Without Return Value","text":"<p>Syntax generally follows <code>void FunctionName(argOne, argTwo)</code> to define the function followed by <code>FuntionName()</code> to call the function.</p> <pre><code>// Usage of void when the function does not return anything\n// In this example, this function prints out the multiplication result of two given numbers\nvoid MultiplyTwoNumbers(int firstNum, int secondNum)\n{\n    // Define variable as integer type\n    long int value;\n    value = firstNum * secondNum;\n    std::cout &lt;&lt; value &lt;&lt; std::endl; \n}\n</code></pre>"},{"location":"programming/cpp/cpp/#multiply-two-numbers-3-and-2","title":"Multiply Two Numbers 3 and 2","text":"<pre><code>MultiplyTwoNumbers(3, 2)\n</code></pre> <pre><code>6\n</code></pre>"},{"location":"programming/cpp/cpp/#multiply-two-numbers-6-and-2","title":"Multiply Two Numbers 6 and 2","text":"<pre><code>MultiplyTwoNumbers(6, 2)\n</code></pre> <pre><code>12\n</code></pre>"},{"location":"programming/cpp/cpp/#aliases-function","title":"Aliases Function","text":"<p>Say we want the variable <code>value</code> to be of type <code>unsigned short int</code> such that it's 2 bytes and can hold 2x the range of values compared to just <code>short int</code>. We can use <code>typedef</code> as an alias.</p> Type Bytes (Size) Range of Values short int 2 -32,768 to 32,767 unsigned short int 2 0 to 65,536 <pre><code>// Usage of void when the function does not return anything\n// In this exmaple, this function prints out the multiplication result of two given numbers\nvoid MultiplyTwoNumbersWithAlias(int firstNum, int secondNum)\n{\n    // Using an alias\n    typedef unsigned short int ushortint;\n    // initializing value variable with ushortint type\n    ushortint value;\n    value = firstNum * secondNum;\n    std::cout &lt;&lt; value &lt;&lt; std::endl; \n}\n</code></pre>"},{"location":"programming/cpp/cpp/#multiply-two-numbers-10-and-10","title":"Multiply Two Numbers 10 and 10","text":"<pre><code>MultiplyTwoNumbersWithAlias(10, 10)\n</code></pre> <pre><code>100\n</code></pre>"},{"location":"programming/cpp/cpp/#multiply-two-numbers-1000-and-65","title":"Multiply Two Numbers 1000 and 65","text":"<pre><code>MultiplyTwoNumbersWithAlias(1000, 65)\n</code></pre> <pre><code>65000\n</code></pre>"},{"location":"programming/cpp/cpp/#multiply-two-numbers-1000-and-67","title":"Multiply Two Numbers 1000 and 67","text":"<ul> <li>Notice how you don't get 67,000? This is because our variable <code>value</code> of <code>ushortint</code> type can only hold values up to the integer 65,536.</li> <li>What this returns is the remainder of 67,000 - 65,536 = 1464</li> </ul> <pre><code>MultiplyTwoNumbersWithAlias(1000, 67)\n</code></pre> <pre><code>1464\n</code></pre> <pre><code>std::cout &lt;&lt; 67 * 1000 - 65536 &lt;&lt; std::endl;\n</code></pre> <pre><code>1464\n</code></pre>"},{"location":"programming/cpp/cpp/#function-with-return-value","title":"Function with Return Value","text":"<p>Unlike functions without return values where we use <code>void</code> to declare the function, here we use <code>int</code> to declare our function that returns values.</p> <pre><code>// In this exmaple, this function returns the value of the multiplication of two numbers\nint MultiplyTwoNumbersNoPrint(int firstNum, int secondNum)\n{\n    // Define variable as integer type\n    long int value;\n    value = firstNum * secondNum;\n    return value;\n}\n</code></pre>"},{"location":"programming/cpp/cpp/#call-function","title":"Call Function","text":"<pre><code>// Declare variable with type\nlong int returnValue;\n// Call function\nreturnValue = MultiplyTwoNumbersNoPrint(10, 2);\n// Print variable\nstd::cout &lt;&lt; returnValue &lt;&lt; std::endl;\n</code></pre> <pre><code>20\n</code></pre>"},{"location":"programming/cpp/cpp/#function-inner-workings","title":"Function Inner Workings","text":"<ul> <li>Essentially our lines of codes translates to instruction pointers with unique memory addresses.</li> <li>Execution of instruction pointers operates on a \"LIFO\" basis, last in first out.<ul> <li>Oversimplifying here, in our example, the last line is taken off first and it follows up</li> </ul> </li> </ul> <pre><code>// Code Space\nint varOneTest = 10; // Instruction pointer 100\nstd::cout &lt;&lt; varOneTest &lt;&lt; std::endl; // Instruction Pointer 102\n</code></pre> <pre><code>10\n\n\n\n\n\n@0x7fa6b7de5460\n</code></pre>"},{"location":"programming/cpp/cpp/#arrays","title":"Arrays","text":"<p>An array contains a sequence of elements with the same data type. </p>"},{"location":"programming/cpp/cpp/#creating-an-array","title":"Creating an Array","text":"<pre><code>// This is how you declare an array of 50 elements each of type double\ndouble DoubleArray[50];\n</code></pre>"},{"location":"programming/cpp/cpp/#accessing-arrays-elements","title":"Accessing Array's Elements","text":""},{"location":"programming/cpp/cpp/#first-element","title":"First Element","text":"<pre><code>// This access the first array \nstd::cout &lt;&lt; DoubleArray[0] &lt;&lt; std::endl;\n</code></pre> <pre><code>0\n</code></pre>"},{"location":"programming/cpp/cpp/#last-element","title":"Last Element","text":"<pre><code>std::cout &lt;&lt; DoubleArray[49] &lt;&lt; std::endl;\n</code></pre> <pre><code>0\n</code></pre>"},{"location":"programming/cpp/cpp/#first-10-elements","title":"First 10 Elements","text":"<pre><code>// In steps of 1\nfor (int i=0; i&lt;10; i++)\n{\n    // This is how you print a mix of characters and declared variables\n    std::cout &lt;&lt; \"Element \" &lt;&lt; i &lt;&lt; \" contains \" &lt;&lt;  DoubleArray[i] &lt;&lt; std::endl;\n}\n</code></pre> <pre><code>Element 0 contains 0\nElement 1 contains 0\nElement 2 contains 0\nElement 3 contains 0\nElement 4 contains 0\nElement 5 contains 0\nElement 6 contains 0\nElement 7 contains 0\nElement 8 contains 0\nElement 9 contains 0\n</code></pre> <pre><code>// In steps of 2\nfor (int i=0; i&lt;10; i+=2)\n{\n    // This is how you print a mix of characters and declared variables\n    std::cout &lt;&lt; \"Element \" &lt;&lt; i &lt;&lt; \" contains \" &lt;&lt;  DoubleArray[i] &lt;&lt; std::endl;\n}\n</code></pre> <pre><code>Element 0 contains 0\nElement 2 contains 0\nElement 4 contains 0\nElement 6 contains 0\nElement 8 contains 0\n</code></pre>"},{"location":"programming/cpp/cpp/#going-beyond-the-arrays-length","title":"Going Beyond The Array's Length","text":"<p>This will return a warning that it's past the end of the array</p> <p><pre><code>std::cout &lt;&lt; DoubleArray[50] &lt;&lt; std::endl;\n</code></pre> <pre><code>input_line_36:2:15: warning: array index 50 is past the end of the array (which contains 50 elements)\n      [-Warray-bounds]\n std::cout &lt;&lt; DoubleArray[50] &lt;&lt; std::endl;\n              ^           ~~\ninput_line_32:3:1: note: array 'DoubleArray' declared here\ndouble DoubleArray[50];\n^\n4.94066e-323\n</code></pre></p>"},{"location":"programming/cpp/cpp/#arrays-with-enumeration","title":"Arrays with Enumeration","text":"<pre><code>enum directionsNew {up, down, left, right, individualDirections};\n\nint directionsArray[individualDirections] = {1, 2, 3, 4};\n\nstd::cout &lt;&lt; \"Up value:\\t\" &lt;&lt; directionsArray[up];\nstd::cout &lt;&lt; \"\\nDown value:\\t\" &lt;&lt; directionsArray[down];\nstd::cout &lt;&lt; \"\\nLeft value:\\t\" &lt;&lt; directionsArray[left];\nstd::cout &lt;&lt; \"\\nRight value:\\t\" &lt;&lt; directionsArray[right];\n// This is the number of elements in the array\nstd::cout &lt;&lt; \"\\nNum value:\\t\" &lt;&lt; sizeof(directionsArray) / sizeof(directionsArray[0]) &lt;&lt; std::endl;\n</code></pre> <pre><code>Up value:   1\nDown value: 2\nLeft value: 3\nRight value:    4\nNum value:  4\n</code></pre>"},{"location":"programming/cpp/cpp/#arrays-with-1-dimension-tensors","title":"Arrays with &gt;1 Dimension (Tensors)","text":""},{"location":"programming/cpp/cpp/#multi-dimension-array-with-numbers","title":"Multi Dimension Array with Numbers","text":"<pre><code>// This is how you declare a multi-dimensional array of 5x5 elements each of type double\ndouble multiDimArray[5][5] = {\n    {1, 2, 3, 4, 5},\n    {2, 2, 3, 4, 5},\n    {3, 2, 3, 4, 5},\n    {4, 2, 3, 4, 5},\n    {5, 2, 3, 4, 5}\n};\n\n// Print each row of our 5x5 multi-dimensional array\nfor (int i=0; i&lt;5; i++)\n{\n    for (int j=0; j&lt;5; j++)\n    {\n        std::cout &lt;&lt; multiDimArray[i][j];\n    };\n    std::cout &lt;&lt; \"\\n\" &lt;&lt; std::endl;\n};\n</code></pre> <pre><code>12345\n\n22345\n\n32345\n\n42345\n\n52345\n</code></pre>"},{"location":"programming/cpp/cpp/#multi-dimension-array-with-characters","title":"Multi Dimension Array with Characters","text":"<pre><code>// This is how you declare a multi-dimensional array of 5x5 elements each of type char\nchar multiDimArrayChars[5][5] = {\n    {'a', 'b', 'c', 'd', 'e'},\n    {'b', 'b', 'c', 'd', 'e'},\n    {'c', 'b', 'c', 'd', 'e'},\n    {'d', 'b', 'c', 'd', 'e'},\n    {'e', 'b', 'c', 'd', 'e'},\n};\n\n// Print each row of our 5x5 multi-dimensional array\nfor (int i=0; i&lt;5; i++)\n{\n    for (int j=0; j&lt;5; j++)\n    {\n        std::cout &lt;&lt; multiDimArrayChars[i][j];\n    };\n    std::cout &lt;&lt; \"\\n\" &lt;&lt; std::endl;\n};\n</code></pre> <pre><code>abcde\n\nbbcde\n\ncbcde\n\ndbcde\n\nebcde\n</code></pre>"},{"location":"programming/cpp/cpp/#copy-arrays","title":"Copy Arrays","text":""},{"location":"programming/cpp/cpp/#copy-number-arrays","title":"Copy Number Arrays","text":"<pre><code>double ArrayNumOne[] = {1, 2, 3, 4, 5};\ndouble ArrayNumTwo[5];\n\n// Use namespace std so it's cleaner\nusing namespace std;\n\n// Copy array with copy()\ncopy(begin(ArrayNumOne), end(ArrayNumOne), begin(ArrayNumTwo));\n\n// Print double type array with copy()\ncopy(begin(ArrayNumTwo), end(ArrayNumTwo), ostream_iterator&lt;double&gt;(cout, \"\\n\"));\n</code></pre> <pre><code>1\n2\n3\n4\n5\n</code></pre>"},{"location":"programming/cpp/cpp/#copy-string-arrays","title":"Copy String Arrays","text":"<pre><code>char ArrayCharOne[] = {'a', 'b', 'c', 'd', 'e'};\nchar ArrayCharTwo[5];\n\n// Use namespace std so it's cleaner\nusing namespace std;\n\n// Copy array with copy()\ncopy(begin(ArrayCharOne), end(ArrayCharOne), begin(ArrayCharTwo));\n\n// Print char type array with copy()\ncopy(begin(ArrayCharTwo), end(ArrayCharTwo), ostream_iterator&lt;char&gt;(cout, \"\\n\"));\n</code></pre> <pre><code>a\nb\nc\nd\ne\n</code></pre>"},{"location":"programming/cpp/cpp/#mathematical-operators","title":"Mathematical Operators","text":"<ul> <li>Add, subtract, multiply, divide and modulus</li> </ul>"},{"location":"programming/cpp/cpp/#add","title":"Add","text":"<pre><code>double addStatement = 5 + 5;\nstd::cout &lt;&lt; addStatement;\n</code></pre> <pre><code>10\n</code></pre>"},{"location":"programming/cpp/cpp/#subtract","title":"Subtract","text":"<pre><code>double subtractStatement = 5 - 5;\nstd::cout &lt;&lt; subtractStatement;\n</code></pre> <pre><code>0\n</code></pre>"},{"location":"programming/cpp/cpp/#divide","title":"Divide","text":"<pre><code>double divideStatement = 5 / 5;\nstd::cout &lt;&lt; divideStatement;\n</code></pre> <pre><code>1\n</code></pre>"},{"location":"programming/cpp/cpp/#multiply","title":"Multiply","text":"<pre><code>double multiplyStatement = 5 * 5;\nstd::cout &lt;&lt; multiplyStatement;\n</code></pre> <pre><code>25\n</code></pre>"},{"location":"programming/cpp/cpp/#modulus","title":"Modulus","text":"<ul> <li>Gets the remainder of the division</li> </ul> <pre><code>double modulusStatement = 8 % 5;\nstd::cout &lt;&lt; modulusStatement;\n</code></pre> <pre><code>3\n</code></pre>"},{"location":"programming/cpp/cpp/#exponent","title":"Exponent","text":"<ul> <li>Base to the power of something, this requires a new package called <code>&lt;cmath&gt;</code> that we want to include.</li> </ul> <pre><code>#include &lt;cmath&gt;\n\nvoid SquareNumber(int baseNum, int exponentNum)\n{\n    // Square the locally scoped variable with 2\n    int squaredNumber;\n    squaredNumber = pow(baseNum, exponentNum);\n    std::cout &lt;&lt; \"Base of 2 with exponent of 2 gives: \" &lt;&lt; squaredNumber &lt;&lt; std::endl;\n}\n\nSquareNumber(2, 2)\n</code></pre> <pre><code>Base of 2 with exponent of 2 gives: 4\n</code></pre>"},{"location":"programming/cpp/cpp/#incrementingdecrementing","title":"Incrementing/Decrementing","text":"<ul> <li>3 ways to do this, from least to most verbose</li> </ul>"},{"location":"programming/cpp/cpp/#methods","title":"Methods","text":""},{"location":"programming/cpp/cpp/#method-1","title":"Method 1","text":"<pre><code>double idx = 1;\nidx++;\nstd::cout &lt;&lt; idx;\n</code></pre> <pre><code>2\n</code></pre>"},{"location":"programming/cpp/cpp/#method-2","title":"Method 2","text":"<pre><code>idx += 1;\nstd::cout &lt;&lt; idx;\n</code></pre> <pre><code>3\n</code></pre>"},{"location":"programming/cpp/cpp/#method-3","title":"Method 3","text":"<pre><code>idx = idx + 1;\nstd::cout &lt;&lt; idx;\n</code></pre> <pre><code>4\n</code></pre>"},{"location":"programming/cpp/cpp/#prefixpostfix","title":"Prefix/Postfix","text":""},{"location":"programming/cpp/cpp/#prefix","title":"Prefix","text":"<ul> <li>This will change both incremented variable and the new variable you assign the incremented variable to</li> <li>Summary: both variables will have the same values</li> </ul> <pre><code>// Instantiate\ndouble a = 1;\ndouble b = 1;\n\n// Print original values\ncout &lt;&lt; \"Old a:\\t\" &lt;&lt; a &lt;&lt; \"\\n\";\ncout &lt;&lt; \"Old b:\\t\" &lt;&lt; b &lt;&lt; \"\\n\";\n\n// Prefix increment\na = ++b;\n\n// Print new values\ncout &lt;&lt; \"New a:\\t\" &lt;&lt; a &lt;&lt; \"\\n\";\ncout &lt;&lt; \"New b:\\t\" &lt;&lt; b &lt;&lt; \"\\n\";\n</code></pre> <pre><code>Old a:  1\nOld b:  1\nNew a:  2\nNew b:  2\n</code></pre>"},{"location":"programming/cpp/cpp/#postfix","title":"Postfix","text":"<ul> <li>This will change only the incremented variable but not the variable it's assigned to</li> <li>Summary: incremented variable will change but not the variable it was assigned to</li> </ul> <pre><code>// Instantiate\ndouble c = 2;\ndouble d = 2;\n\n// Print original values\ncout &lt;&lt; \"Old c:\\t\" &lt;&lt; c &lt;&lt; \"\\n\";\ncout &lt;&lt; \"Old d:\\t\" &lt;&lt; d &lt;&lt; \"\\n\";\n\n// Prefix increment\nc = d--;\n\n// Print new values, notice how only d decremented? c which is what d is assigned to doesn't change.\ncout &lt;&lt; \"New c:\\t\" &lt;&lt; c &lt;&lt; \"\\n\";\ncout &lt;&lt; \"New d:\\t\" &lt;&lt; d &lt;&lt; \"\\n\";\n</code></pre> <pre><code>Old c:  2\nOld d:  2\nNew c:  2\nNew d:  1\n</code></pre>"},{"location":"programming/cpp/cpp/#conditional-statements","title":"Conditional Statements","text":""},{"location":"programming/cpp/cpp/#if","title":"If","text":"<pre><code>int maxValue = 10;\n\n// Increment till 10\nfor (int i=0; i&lt;=10; i+=2)\n{\n    // Stop if the number reaches 10 (inclusive)\n    if (i == maxValue)\n    {\n        cout &lt;&lt; \"Reached max value!\";\n        cout &lt;&lt; \"\\nValue is \" &lt;&lt; i &lt;&lt; endl;\n    };\n};\n</code></pre> <pre><code>Reached max value!\nValue is 10\n</code></pre>"},{"location":"programming/cpp/cpp/#else","title":"Else","text":"<pre><code>int newMaxValue = 20;\n\n// Increment till 10\nfor (int i=0; i&lt;=10; i+=2)\n{\n    // Stop if the number reaches 10 (inclusive)\n    if (i == newMaxValue)\n    {\n        cout &lt;&lt; \"Reached max value!\";\n        cout &lt;&lt; \"\\nValue is \" &lt;&lt; i &lt;&lt; endl;\n    }\n    // Else print current value\n    else\n    {\n        cout &lt;&lt; \"\\nCurrent Value is \" &lt;&lt; i &lt;&lt; endl;\n    }\n}\n</code></pre> <pre><code>Current Value is 0\n\nCurrent Value is 2\n\nCurrent Value is 4\n\nCurrent Value is 6\n\nCurrent Value is 8\n\nCurrent Value is 10\n</code></pre>"},{"location":"programming/cpp/cpp/#logical-operators","title":"Logical Operators","text":""},{"location":"programming/cpp/cpp/#and","title":"And","text":"<pre><code>int varOneNew = 10;\nint varTwo = 10;\nint varCheckOne = 10;\nint varCheckTwo = 5;\n\n// This should print out\nif ((varOneNew == varCheckOne) &amp;&amp; (varTwo == varCheckOne))\n{\n    std::cout &lt;&lt; \"Both values equal to 10!\" &lt;&lt; std::endl;\n}\n\n// This should not print out as varTwo does not equal to 5\nif ((varOneNew == varCheckOne) &amp;&amp; (varTwo == varCheckTwo))\n{\n    std::cout &lt;&lt; \"VarOneNew equals to 10, VarTwo equals to 5\" &lt;&lt; std::endl;\n}\n</code></pre> <pre><code>Both values equal to 10!\n</code></pre>"},{"location":"programming/cpp/cpp/#or","title":"Or","text":"<pre><code>// On the contrary, this exact same statement would print out\n// as VarOne is equal to 10 and we are using an OR operator\nif ((varOneNew == varCheckOne) || (varTwo == varCheckTwo))\n{\n    std::cout &lt;&lt; \"VarOneNew equals to 10 or VarTwo equals to 5\" &lt;&lt; std::endl;\n}\n</code></pre> <pre><code>VarOneNew equals to 10 or VarTwo equals to 5\n</code></pre>"},{"location":"programming/cpp/cpp/#not","title":"Not","text":"<pre><code>// This would print out as VarTwo is not equal to 5\nif (varTwo != varCheckTwo)\n{\n    std::cout &lt;&lt; \"VarTwo (10) is not equal to VarCheckTwo (5).\" &lt;&lt; std::endl;\n}\n</code></pre> <pre><code>VarTwo (10) is not equal to VarCheckTwo (5).\n</code></pre>"},{"location":"programming/cpp/cpp/#getting-user-input","title":"Getting User Input","text":"<pre><code>using namespace std;\nlong double inputOne, inputTwo;\ncout &lt;&lt; \"This program multiplies 2 given numbers\\n\";\ncout &lt;&lt; \"Enter first number: \\n\";\ncin &gt;&gt; inputOne;\ncout &lt;&lt; \"Enter second number: \\n\";\ncin &gt;&gt; inputTwo;\ncout &lt;&lt; \"Multiplication value: \" &lt;&lt; inputOne * inputTwo &lt;&lt; endl;\n</code></pre> <pre><code>This program multiplies 2 given numbers\nEnter first number: \n10\nEnter second number: \n10\nMultiplication value: 100\n</code></pre>"},{"location":"programming/cpp/cpp/#loops","title":"Loops","text":""},{"location":"programming/cpp/cpp/#for-loop","title":"For Loop","text":"<pre><code>for (int i=0; i&lt;10; i+=1)\n{\n    cout &lt;&lt; \"Value of i is: \" &lt;&lt; i &lt;&lt; endl;\n}\n</code></pre> <pre><code>Value of i is: 0\nValue of i is: 1\nValue of i is: 2\nValue of i is: 3\nValue of i is: 4\nValue of i is: 5\nValue of i is: 6\nValue of i is: 7\nValue of i is: 8\nValue of i is: 9\n</code></pre>"},{"location":"programming/cpp/cpp/#while-loop","title":"While Loop","text":"<pre><code>int idxWhile = 0;\n\nwhile (idxWhile &lt; 10)\n{\n    idxWhile += 1;\n    cout &lt;&lt; \"Value of while loop i is: \" &lt;&lt; idxWhile &lt;&lt; endl;\n}\n</code></pre> <pre><code>Value of while loop i is: 1\nValue of while loop i is: 2\nValue of while loop i is: 3\nValue of while loop i is: 4\nValue of while loop i is: 5\nValue of while loop i is: 6\nValue of while loop i is: 7\nValue of while loop i is: 8\nValue of while loop i is: 9\nValue of while loop i is: 10\n</code></pre>"},{"location":"programming/cpp/cpp/#while-loop-with-continuebreak","title":"While Loop with Continue/Break","text":"<pre><code>int idxWhileNew = 0;\n\nwhile (idxWhileNew &lt; 100)\n{\n    idxWhileNew += 1;\n    cout &lt;&lt; \"Value of while loop i is: \" &lt;&lt; idxWhile &lt;&lt; endl;\n\n    if (idxWhileNew == 10)\n    {\n        cout &lt;&lt; \"Max value of 10 reached!\" &lt;&lt; endl;\n        break;\n    }\n}\n</code></pre> <pre><code>Value of while loop i is: 10\nValue of while loop i is: 10\nValue of while loop i is: 10\nValue of while loop i is: 10\nValue of while loop i is: 10\nValue of while loop i is: 10\nValue of while loop i is: 10\nValue of while loop i is: 10\nValue of while loop i is: 10\nValue of while loop i is: 10\nMax value of 10 reached!\n</code></pre>"},{"location":"programming/electron/electron/","title":"Electron","text":""},{"location":"programming/electron/electron/#why-electron","title":"Why Electron","text":"<p>We choose to cover electron as you can easily use it as a front-end application across any platforms (Windows, MacOS, Linux or even a mobile application) for your AI applications.</p>"},{"location":"programming/electron/electron/#installation-of-electron","title":"Installation of Electron","text":"<pre><code>npm i -D electron@latest\n</code></pre>"},{"location":"programming/electron/electron/#creating-electron-project","title":"Creating Electron Project","text":""},{"location":"programming/electron/electron/#critical-files","title":"Critical Files","text":"<p>You should have 3 base files <code>package.json</code>, <code>main.js</code> and <code>index.html</code> to have a basic application.</p> <pre><code>mkdir app\ncd app\n\nnpm init\n\ntouch main.js\ntouch index.html\n</code></pre>"},{"location":"programming/electron/electron/#edit-packagejson","title":"Edit package.json","text":"<ul> <li>When you run <code>npm init</code>, it should create a <code>package.json</code> file. But we need to make some tiny changes to leverage on electron.</li> <li>Key fields<ul> <li><code>name</code>: name of your app, can be anything</li> <li><code>version</code>: version of your app, can be anything</li> <li><code>main</code>: main javascript file, we recommend using <code>main.js</code></li> <li><code>scripts</code>: here you want to copy the whole <code>scripts</code> section to leverage on electron</li> <li><code>devDependencies</code>: electron version required</li> </ul> </li> </ul> <pre><code>{\n  \"name\": \"dlw\",\n  \"version\": \"0.1.0\",\n  \"main\": \"main.js\",\n  \"scripts\": {\n    \"start\": \"electron .\"\n  },\n  \"devDependencies\": {\n    \"electron\": \"^6.0.8\"\n  }\n}\n</code></pre>"},{"location":"programming/electron/electron/#edit-mainjs","title":"Edit main.js","text":"<p>This beautiful boilerplate code is provided by Electron, full credits to them. </p> <pre><code>const { app, BrowserWindow } = require('electron')\n\n// Keep a global reference of the window object, if you don't, the window will\n// be closed automatically when the JavaScript object is garbage collected.\nlet win\n\nfunction createWindow () {\n  // Create the browser window.\n  win = new BrowserWindow({\n    width: 800,\n    height: 600,\n    webPreferences: {\n      nodeIntegration: true\n    }\n  })\n\n  // and load the index.html of the app.\n  win.loadFile('index.html')\n\n  // Open the DevTools.\n  // win.webContents.openDevTools()\n\n  // Emitted when the window is closed.\n  win.on('closed', () =&gt; {\n    // Dereference the window object, usually you would store windows\n    // in an array if your app supports multi windows, this is the time\n    // when you should delete the corresponding element.\n    win = null\n  })\n}\n\n// This method will be called when Electron has finished\n// initialization and is ready to create browser windows.\n// Some APIs can only be used after this event occurs.\napp.on('ready', createWindow)\n\n// Quit when all windows are closed.\napp.on('window-all-closed', () =&gt; {\n  // On macOS it is common for applications and their menu bar\n  // to stay active until the user quits explicitly with Cmd + Q\n  if (process.platform !== 'darwin') {\n    app.quit()\n  }\n})\n\napp.on('activate', () =&gt; {\n  // On macOS it's common to re-create a window in the app when the\n  // dock icon is clicked and there are no other windows open.\n  if (win === null) {\n    createWindow()\n  }\n})\n\n// In this file you can include the rest of your app's specific main process\n// code. You can also put them in separate files and require them here.\n</code></pre>"},{"location":"programming/electron/electron/#edit-indexhtml","title":"Edit index.html","text":"<p>I modified this script from electron's boilerplate code where it will display critical dependencies' versions for your node, chrome and electron.</p> <pre><code>&lt;!DOCTYPE html&gt;\n&lt;html&gt;\n\n  &lt;head&gt;\n    &lt;meta charset=\"UTF-8\"&gt;\n    &lt;title&gt;Dashboard&lt;/title&gt;\n  &lt;/head&gt;\n\n  &lt;body&gt;\n    &lt;h1&gt;Dashboard&lt;/h1&gt;\n\n    &lt;h2&gt;Environment&lt;/h2&gt;\n        &lt;br/&gt;\n        Node: &lt;script&gt;document.write(process.versions.node)&lt;/script&gt;\n\n        &lt;br/&gt;\n        Chrome: &lt;script&gt;document.write(process.versions.chrome)&lt;/script&gt;\n\n        &lt;br/&gt;\n        Electron: &lt;script&gt;document.write(process.versions.electron)&lt;/script&gt;\n  &lt;/body&gt;\n&lt;/html&gt;\n</code></pre>"},{"location":"programming/electron/electron/#starting-app","title":"Starting App","text":"<p>This will start your electron application.</p> <pre><code>npm start\n</code></pre>"},{"location":"programming/electron/electron/#packaging-electron-app","title":"Packaging Electron App","text":""},{"location":"programming/electron/electron/#wine","title":"Wine","text":"<p>The reason for installing Wine is being able to package Electron applications for the Windows platform, creating the executable file <code>app.exe</code> like any other application on Windows. The final aim of our tutorial is to package the app for Windows, MacOS and Ubuntu.</p>"},{"location":"programming/electron/electron/#installation-of-wine","title":"Installation of Wine","text":"<p>This assumes installation on Ubuntu 16.04 <code>xenial</code>, if you're on Ubuntu 18.04 or 19.04, change to <code>bionic</code> and <code>disco</code> respectively.</p> <p>Also, this works on 64-bit system architecture.</p> <pre><code>cd ~\nwget -qO - https://dl.winehq.org/wine-builds/winehq.key | sudo apt-key add -\nsudo apt-add-repository 'deb https://dl.winehq.org/wine-builds/ubuntu/ xenial main'\nsudo apt-get update\nsudo apt-get install --install-recommends winehq-stable\n\nsudo chown root:root ~/.wine\n</code></pre>"},{"location":"programming/electron/electron/#check-wine-version","title":"Check Wine Version","text":"<pre><code>wine --version\n</code></pre>"},{"location":"programming/electron/electron/#packaging-windows-application","title":"Packaging Windows Application","text":"<p>This packages the application churning the necessary files and the executable <code>app.exe</code> for windows 64 bit.</p> <pre><code>electron-packager ./app app --platform=win32 --arch=x64\n</code></pre>"},{"location":"programming/electron/electron/#python-scripts","title":"Python Scripts","text":""},{"location":"programming/electron/electron/#installing-python-node-package","title":"Installing Python Node Package","text":"<p>So we want to easily create Python scripts and run through Javascript in the Electron application. This can be done via <code>python-shell</code> npm package.</p> <pre><code>sudo npm install --save python-shell \n</code></pre>"},{"location":"programming/electron/electron/#creating-hello-from-js","title":"Creating \"Hello from JS\"","text":""},{"location":"programming/electron/electron/#javascript","title":"Javascript","text":"<p>In your <code>main.js</code> file, you would want to add the following code. </p> <p>This leverages on the <code>python-shell</code> package to send a message to <code>hello_world.py</code> and receive the message subsequently.</p> <pre><code>// Start Python shell\nlet {PythonShell} = require('python-shell')\n\n// Start shell for specific script for communicating\nlet pyshell = new PythonShell('./scripts/hello_world.py');\n\n// Send a message to the Python script via stdin\npyshell.send('Hello from JS');\n\n// Receive message from Python script\npyshell.on('message', function (message) {\n  console.log(message);\n\n});\n\n// End the input stream and allow the process to exit\npyshell.end(function (err, code, signal) {\n  if (err) throw err;\n//  console.log('The exit code was: ' + code);\n//  console.log('The exit signal was: ' + signal);\n  console.log('finished');\n});\n</code></pre>"},{"location":"programming/electron/electron/#python","title":"Python","text":"<p>Create a folder <code>scripts</code> to hold all your Python scripts. Then create a Python file named <code>hello_world.py</code> with the following content.</p> <pre><code>import sys\n\nmsg_from_js = sys.stdin.read()\n\nprint(msg_from_js)\n</code></pre>"},{"location":"programming/electron/electron/#run-app","title":"Run App","text":"<p>Run via <code>npm start</code> and you'll see this in your bash output. Viola! We managed to call <code>hello_world.py</code> via <code>main.js</code> through the <code>python-shell</code> package. Next task, we will be passing this message to <code>index.html</code>.</p> <pre><code>Hello from JS\n\nfinished\n</code></pre>"},{"location":"programming/javascript/javascript/","title":"Javascript","text":"<p>Run Jupyter Notebook</p> <p>You can run the code for this section in this jupyter notebook link.</p>"},{"location":"programming/javascript/javascript/#installation-of-ijavascript","title":"Installation of iJavascript","text":"<p>This will enable the Javascript kernel to be installed in your jupyter notebook kernel list so you can play with Javascript easily.</p> <p>For now, the installation requires the ancient Python 2.7. Hopefully this changes in the future.</p> <pre><code>conda create -n py27 python=2.7\nconda activate py27\nsudo npm install -g --unsafe-perm ijavascript\nconda install jupyter\nijsinstall\njupyter notebook\n</code></pre>"},{"location":"programming/javascript/javascript/#variables-constants","title":"Variables &amp; Constants","text":""},{"location":"programming/javascript/javascript/#variable","title":"Variable","text":""},{"location":"programming/javascript/javascript/#declaring-variable","title":"Declaring Variable","text":"<pre><code>var randomNumber = 1.52\n</code></pre>"},{"location":"programming/javascript/javascript/#read-out-variable","title":"Read out Variable","text":"<pre><code>// Check value\nrandomNumber\n</code></pre> <pre><code>1.52\n</code></pre>"},{"location":"programming/javascript/javascript/#read-out-via-console","title":"Read out via Console","text":"<pre><code>// Using the console via log and error\nconsole.log(randomNumber)\nconsole.error(randomNumber)\n</code></pre> <pre><code>1.52\n\n\n1.52\n</code></pre>"},{"location":"programming/javascript/javascript/#constants","title":"Constants","text":""},{"location":"programming/javascript/javascript/#declaring-constant","title":"Declaring Constant","text":"<p>This is a weird one, because in Javascript when you declare a constant, you are essentially unable to overwrite the constant subsequently. It's useful when you want a fixed value.</p> <pre><code>const cannotOverwriteNumber = 2.22\n</code></pre>"},{"location":"programming/javascript/javascript/#read-out-constant","title":"Read out Constant","text":"<pre><code>cannotOverwriteNumber\n</code></pre> <pre><code>2.22\n</code></pre>"},{"location":"programming/javascript/javascript/#overwrite-constant-error","title":"Overwrite Constant (Error)","text":"<p>This will throw an error because this constant has been declared once and you cannot do it again.</p> <pre><code>const cannotOverwriteNumber = 1.52\n</code></pre> <pre><code>evalmachine.&lt;anonymous&gt;:1\n\nconst cannotOverwriteNumber = 1.52\n\n^\n\n\n\nSyntaxError: Identifier 'cannotOverwriteNumber' has already been declared\n\n    at evalmachine.&lt;anonymous&gt;:1:1\n\n    at Script.runInThisContext (vm.js:122:20)\n\n    at Object.runInThisContext (vm.js:329:38)\n\n    at run ([eval]:1054:15)\n\n    at onRunRequest ([eval]:888:18)\n\n    at onMessage ([eval]:848:13)\n\n    at process.emit (events.js:198:13)\n\n    at emit (internal/child_process.js:832:12)\n\n    at process._tickCallback (internal/process/next_tick.js:63:19)\n</code></pre> <pre><code>\n</code></pre>"},{"location":"programming/numpycupy/linalg_numpy_cupy/","title":"Linear Algebra with NumPy and CuPy","text":"<p>In this section, we will be covering linear algebra and using numpy for CPU-based matrix manipulation and cupy for GPU-based matrix manipulation.</p>"},{"location":"programming/numpycupy/linalg_numpy_cupy/#types-of-matrices","title":"Types of Matrices","text":"<ul> <li>M x N Matrix<ul> <li>M rows and N columns</li> </ul> </li> <li>1 x N Matrix (Row Vector)<ul> <li>1 row and N columns</li> </ul> </li> <li>M x 1 Matrix (Column Vector)<ul> <li>M rows and 1 column</li> </ul> </li> <li>1 x 1 Matrix (Scalar)<ul> <li>1 row and 1 column</li> </ul> </li> <li>N x M Matrix from M x N Matrix (Transposed Matrix)<ul> <li>Swapping M rows to columns and N columns to rows where matrix \\(A\\) then becomes transposed matrix \\(A\\prime\\)</li> </ul> </li> <li>Symmetric Matrix<ul> <li>M x N Matrix is equal to N x M Transposed Matrix</li> </ul> </li> <li>M x M Matrix (Square Matrix)<ul> <li>M rows = N columns</li> </ul> </li> <li>Diagonal Matrix<ul> <li>all matrix elements are zeroes except for the diagonal</li> </ul> </li> <li>Identity matrix<ul> <li>all matrix elements are zeroes except for the diagonals being 1's </li> </ul> </li> </ul>"},{"location":"programming/numpycupy/linalg_numpy_cupy/#multiplication-and-subtraction","title":"Multiplication and Subtraction","text":"<ul> <li>Element-wise Matrix Addition/Subtraction<ul> <li>add/subtract every individual element \\(a_{ij}\\) of the first matrix \\(A_{M \\times N}\\) with \\(b_{ij}\\) of the second matrix \\(B_{M \\times N}\\)</li> </ul> </li> <li>Scalar Multiplication with Matrix<ul> <li>multiply scalar \\(\\lambda\\) with every individual element \\(a_{ij}\\) of the matrix \\(A_{M \\times N}\\)</li> </ul> </li> <li>Matrix Multiplication<ul> <li>can only be done where \\(A_{M \\times N}\\) and \\(B_{N \\times O}\\) have a coinciding dimension \\(N\\) which is the same to yield a new matrix \\(C_{N \\times O}\\) of \\(N \\times O\\) dimension</li> <li>new element \\(c_{ij} = \\sum _{q=1} ^N a_{iq} b_{qj}\\)</li> <li>essentially the first row and first column element of the new matrix is equal to the summation element-wise multiplication of the first row of matrix \\(A\\) with the first column of matrix \\(B\\)</li> </ul> </li> </ul>"},{"location":"programming/numpycupy/linalg_numpy_cupy/#determinant-of-square-matrix","title":"Determinant of Square Matrix","text":""},{"location":"programming/numpycupy/linalg_numpy_cupy/#determinant-of-2-x-2-square-matrix","title":"Determinant of 2 x 2 Square Matrix","text":"<p>The determinant of a 2x2 square matrix can be derived by multiplying all the elements on the main diagonal and subtracting the multiplication of all the elements in the other diagonal which will result in a scalar value.</p> \\[ A = \\begin{bmatrix}a &amp; b\\\\c &amp; d\\end{bmatrix} \\] \\[\\det(A) = | A | =  ad - bc \\]"},{"location":"programming/numpycupy/linalg_numpy_cupy/#determinant-of-3-x-3-square-matrix","title":"Determinant of 3 x 3 Square Matrix","text":"\\[ \\begin{bmatrix} a &amp; b &amp;c \\\\ d&amp; e &amp;f \\\\ g&amp; h &amp;i \\end{bmatrix} \\\\ = a (-1)^{(1+1)}\\det \\begin{bmatrix} e &amp; f\\\\ h &amp; i \\end{bmatrix} + b (-1)^{(1+2)} \\det \\begin{bmatrix} d &amp; f\\\\ g &amp; i \\end{bmatrix} + c (-1)^{(1+3)} \\det \\begin{bmatrix} d &amp; e\\\\ g &amp; h \\end{bmatrix} \\]"},{"location":"programming/numpycupy/linalg_numpy_cupy/#determinant-types","title":"Determinant Types","text":"<ul> <li>singular matrix: \\(\\det(A) = 0\\) where at least 2 rows/columns are linearly dependent</li> <li>non-singular: \\(\\det(A) \\neq 0\\) where no rows/columns are linearly dependent (matrix has full rank)</li> </ul>"},{"location":"programming/numpycupy/linalg_numpy_cupy/#inverse-of-a-non-singular-square-matrix","title":"Inverse of a Non-Singular Square Matrix","text":""},{"location":"programming/numpycupy/linalg_numpy_cupy/#unique-property","title":"Unique Property","text":"<p>The inverse can be calculated only if it is a non-singular square matrix hence it is useful to first calculate the determinant. Also the inverse of the matrix yields a useful and unique property where</p> \\[ AA^{-1} = A^{-1}A = I \\]"},{"location":"programming/numpycupy/linalg_numpy_cupy/#inverse-and-identity-of-2-x-2-non-singular-square-matrix","title":"Inverse and Identity of 2 x 2 Non-Singular Square Matrix","text":"\\[ A=\\begin{bmatrix}a &amp; b \\\\c &amp; d \\end{bmatrix} \\] \\[ A^{-1}=\\frac{1}{ad-bc}\\begin{bmatrix}d &amp; -b \\\\-c &amp; a \\end{bmatrix} \\] \\[ \\begin{array}{lcl}AA^{-1}&amp;=&amp;\\begin{bmatrix}a &amp; b \\\\c &amp; d \\end{bmatrix}\\frac{1}{ad-bc}\\begin{bmatrix}d &amp; -b \\\\-c &amp; a \\end{bmatrix}\\\\ &amp;=&amp;\\frac{1}{ad-bc}\\begin{bmatrix}ad+b(-c) &amp; a(-b)+b(a) \\\\cd+d(-c) &amp; c(-b)+d(a) \\end{bmatrix}\\\\ &amp;=&amp;\\begin{bmatrix}1 &amp; 0\\\\0 &amp; 1 \\end{bmatrix}\\end{array} \\]"},{"location":"programming/numpycupy/linalg_numpy_cupy/#linear-system-of-equations-in-matrices","title":"Linear System of Equations in Matrices","text":"<p>We can express many linear equations in the form of matrices for example \\(AX= B\\)</p> <p>Where we have \\(A\\) representing our parameters, \\(B\\) representing our input variables and \\(B\\) representing our constant/bias variables.</p> \\[ A = \\begin{bmatrix} a_{11} &amp; a_{12} &amp; \\cdots &amp; a_{1N} \\\\ a_{21} &amp; a_{22} &amp; \\cdots &amp; a_{2N} \\\\ \\vdots &amp; \\vdots &amp; \\ddots &amp; \\vdots \\\\ a_{M1} &amp; a_{M2} &amp; \\cdots &amp; a_{MN} \\end{bmatrix}, X = \\begin{bmatrix} x_1 \\\\ x_2 \\\\ \\vdots \\\\ x_M \\end{bmatrix}, B = \\begin{bmatrix} b_1 \\\\ b_2 \\\\ \\vdots \\\\ b_M \\end{bmatrix} \\] <p>If we find \\(\\det (A)\\) to be non-zero where the matrix is a non-singular matrix, the system of equations would have a unique solution. Hence, given \\(A\\) and \\(B\\), we can find out the unique solution in \\(B\\).</p> \\[  AX = B \\\\ A^{-1}AX = A^{-1}B \\\\ IX = A^{-1}B \\\\ X = A^{-1}B \\]"},{"location":"programming/numpycupy/linalg_numpy_cupy/#solving-system-of-equations-for-square-and-non-singular","title":"Solving System of Equations for Square and Non-Singular","text":"<p>If the matrix A is square and \\(det(A) \\neq 0\\) (non-singular, inverse matrix can be calculated), then Cramer's rule can be applied to solve the linear system of equations.</p>"},{"location":"programming/numpycupy/linalg_numpy_cupy/#process-for-a-2-x-2-non-singular-matrix-a","title":"Process for a 2 x 2 Non-Singular Matrix A","text":"<ul> <li>Replace First Column of \\(A\\) with \\(B\\) to get \\(A_1\\) $$ x_1 = \\frac{\\det(A_1)}{\\det(A)} $$</li> <li>Replace Second Column of \\(A\\) with \\(B\\) to get \\(A_2\\) $$ x_2 = \\frac{\\det(A_2)}{\\det(A)} $$</li> <li>Repeat till all columns covered for bigger matrices</li> </ul>"},{"location":"programming/numpycupy/linalg_numpy_cupy/#eigenvalue-characteristic-root-and-eigenvalue-characteristic-value","title":"Eigenvalue (characteristic root) and eigenvalue (characteristic value)","text":"<p>Given a square matrix \\(A\\), eigenvalue \\(\\lambda\\) and eigenvector \\(v\\) where \\(v \\neq 0\\), we have:</p> \\[ A v = \\lambda v \\\\ A v = \\lambda I v \\\\ Av - \\lambda I v = 0 \\\\ (A - \\lambda I)v = 0 \\\\ \\] <p>Since  \\(v \\neq 0\\) then we have characteristic matrix \\((A - \\lambda I) = 0\\).</p>"},{"location":"programming/numpycupy/linalg_numpy_cupy/#solving-eigens","title":"Solving eigens","text":"<ul> <li>We can solve for the determinant of the characteristic matrix (characteristic polynomial) \\(|A - \\lambda I| = 0\\) through this characteristic equation.<ul> <li>We will get multiple values of \\(\\lambda\\)</li> </ul> </li> <li>Substitute \\(\\lambda\\) into \\((A - \\lambda I)v = 0\\), solve for \\(v\\)<ul> <li>If infinite solution (no constant values for x and y), impose uniqueness with \\(v\\prime v  = 1\\) in this case \\(\\begin{bmatrix} x &amp; y \\end{bmatrix} \\begin{bmatrix} x \\\\ y \\end{bmatrix} = x^2 + y^2 = 1\\)</li> <li>Substitute y into \\(v\\prime v  = 1\\) to solve for x, then solve for y</li> <li>Substitute x and y to solve for \\(v\\)</li> </ul> </li> <li>Simple eigenvector \\(\\lambda _1 \\neq \\lambda _2\\) </li> <li>Repeated/double eigenvector \\(\\lambda _1 = \\lambda _2\\)</li> </ul>"},{"location":"programming/numpycupy/linalg_numpy_cupy/#properties-of-eigenvalue-and-eigenvector","title":"Properties of eigenvalue and eigenvector","text":"<ul> <li>If \\(|A - \\lambda I| = 0\\), singular therefore infinite solutions, hence:<ul> <li>\\(\\lambda \\gt 0\\): positive definite</li> <li>\\(\\lambda \\ge 0\\): positive semi-definite</li> <li>\\(\\lambda \\lt 0\\): negative definite</li> <li>\\(\\lambda \\le 0\\): negative semi-definite</li> </ul> </li> </ul>"},{"location":"programming/plotting/intro/","title":"Plotting","text":"Packages you will learn to use <ol> <li>Fast professional static plots: ggplot2 and rpy2</li> <li>Complicated static plots: matplotlib</li> <li>Live stream and/or interactive plots: plotly</li> <li>Dashboard: streamlit</li> </ol> <p>Typically with tremendous mastery of matplotlib, you will be able to create any good looking plots static or interactive. However, it takes many more lines of code and is relatively complicated.</p> <p>Thus, before we go there, we will be covering the fastest way to create professional looking static plots with rpy2 and ggplot rapidly. It is unconventional but it yields the fastest result for time-sensitive learners. We'll then move backwards to then create more complicated plots with matplotlib. Subsequently we'll create live stream and interactive plots with plotly. Finally we will be bringing everything together with streamlit as a dashboard.</p> <p>As usual, you can go freestyle and just use any of our open-source material you think is useful and you do not need to follow the sequence we are suggesting.</p>"},{"location":"programming/python/python/","title":"Python","text":""},{"location":"programming/python/python/#lists","title":"Lists","text":""},{"location":"programming/python/python/#creating-list-manual-fill","title":"Creating List: Manual Fill","text":"<pre><code>lst = [0, 1, 2 ,3]\nprint(lst)\n</code></pre> <pre><code>[0, 1, 2, 3]\n</code></pre>"},{"location":"programming/python/python/#creating-list-list-comprehension","title":"Creating List: List Comprehension","text":"<pre><code>lst = [i for i in range(4)]\nprint(lst)\n</code></pre> <pre><code>[0, 1, 2, 3]\n</code></pre>"},{"location":"programming/python/python/#joining-list-with-blanks","title":"Joining List with Blanks","text":"<pre><code># To use .join(), your list needs to be of type string\nlst_to_string = list(map(str, lst))\n\n# Join the list of strings\nlst_join = ' '.join(lst_to_string)\nprint(lst_join)\n</code></pre> <pre><code>0 1 2 3\n</code></pre>"},{"location":"programming/python/python/#joining-list-with-comma","title":"Joining List with Comma","text":"<pre><code># Join the list of strings\nlst_join = ', '.join(lst_to_string)\nprint(lst_join)\n</code></pre> <pre><code>0, 1, 2, 3\n</code></pre>"},{"location":"programming/python/python/#checking-lists-equal-method-1","title":"Checking Lists Equal: Method 1","text":"<p>Returns <code>True</code> if equal, and <code>False</code> if unequal</p> <pre><code>lst_unequal = [1, 1, 2, 3, 4, 4]\nlst_equal = [0, 0, 0, 0, 0, 0]\n\nprint('-'*50)\nprint('Unequal List')\nprint('-'*50)\n\nprint(lst_unequal[1:])\nprint(lst_unequal[:-1])\nbool_equal = lst_unequal[1:] == lst_unequal[:-1]\nprint(bool_equal)\n\nprint('-'*50)\nprint('Equal List')\nprint('-'*50)\n\nprint(lst_equal[1:])\nprint(lst_equal[:-1])\nbool_equal = lst_equal[1:] == lst_equal[:-1]\nprint(bool_equal)\n</code></pre> <pre><code>--------------------------------------------------\nUnequal List\n--------------------------------------------------\n[1, 2, 3, 4, 4]\n[1, 1, 2, 3, 4]\nFalse\n--------------------------------------------------\nEqual List\n--------------------------------------------------\n[0, 0, 0, 0, 0]\n[0, 0, 0, 0, 0]\nTrue\n</code></pre>"},{"location":"programming/python/python/#checking-lists-equal-method-2","title":"Checking Lists Equal: Method 2","text":"<p>Returns <code>True</code> if equal, and <code>False</code> if unequal. Here, <code>all</code> essentially checks that there is no <code>False</code> in the list. </p> <pre><code>print('-'*50)\nprint('Unequal List')\nprint('-'*50)\n\nlst_check = [i == lst_unequal[0] for i in lst_unequal]\nbool_equal = all(lst_check)\nprint(bool_equal)\n\nprint('-'*50)\nprint('Equal List')\nprint('-'*50)\n\nlst_check = [i == lst_equal[0] for i in lst_equal]\nbool_equal = all(lst_check)\nprint(bool_equal)\n</code></pre> <pre><code>--------------------------------------------------\nUnequal List\n--------------------------------------------------\nFalse\n--------------------------------------------------\nEqual List\n--------------------------------------------------\nTrue\n</code></pre>"},{"location":"programming/python/python/#sets","title":"Sets","text":""},{"location":"programming/python/python/#removing-duplicate-from-list","title":"Removing Duplicate from List","text":"<p>Sets can be very useful for quickly removing duplicates from a list, essentially finding unique values</p> <pre><code>lst_one = [1, 2, 3, 5]\nlst_two = [1, 1, 2, 4]\nlst_both = lst_one + lst_two\nlst_no_duplicate = list(set(lst_both))\n\nprint(f'Original Combined List {lst_both}')\nprint(f'No Duplicated Combined List {lst_no_duplicate}')\n</code></pre> <pre><code>Original Combined List [1, 2, 3, 5, 1, 1, 2, 4]\nNo Duplicated Combined List [1, 2, 3, 4, 5]\n</code></pre>"},{"location":"programming/python/python/#lambda-map-filter-reduce-partial","title":"Lambda, map, filter, reduce, partial","text":""},{"location":"programming/python/python/#lambda","title":"Lambda","text":"<p>The syntax is simple <code>lambda your_variables: your_operation</code></p>"},{"location":"programming/python/python/#add-function","title":"Add Function","text":"<pre><code>add = lambda x, y: x + y\nadd(2, 3)\n</code></pre> <pre><code>5\n</code></pre>"},{"location":"programming/python/python/#multiply-function","title":"Multiply Function","text":"<pre><code>multiply = lambda x, y: x * y \nmultiply(2, 3)\n</code></pre> <pre><code>6\n</code></pre>"},{"location":"programming/python/python/#map","title":"Map","text":""},{"location":"programming/python/python/#create-list","title":"Create List","text":"<pre><code>lst = [i for i in range(11)]\nprint(lst)\n</code></pre> <pre><code>[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n</code></pre>"},{"location":"programming/python/python/#map-square-function-to-list","title":"Map Square Function to List","text":"<pre><code>square_element = map(lambda x: x**2, lst)\n\n# This gives you a map object\nprint(square_element)\n\n# You need to explicitly return a list\nprint(list(square_element))\n</code></pre> <pre><code>&lt;map object at 0x7f08c8620438&gt;\n[0, 1, 4, 9, 16, 25, 36, 49, 64, 81, 100]\n</code></pre>"},{"location":"programming/python/python/#create-multiple-list","title":"Create Multiple List","text":"<pre><code>lst_1 = [1, 2, 3, 4]\nlst_2 = [2, 4, 6, 8]\nlst_3 = [3, 6, 9, 12]\n</code></pre>"},{"location":"programming/python/python/#map-add-function-to-multiple-lists","title":"Map Add Function to Multiple Lists","text":"<pre><code>add_elements = map(lambda x, y, z : x + y + z, lst_1, lst_2, lst_3)\nprint(list(add_elements))\n</code></pre> <pre><code>[6, 12, 18, 24]\n</code></pre>"},{"location":"programming/python/python/#filter","title":"Filter","text":""},{"location":"programming/python/python/#create-list_1","title":"Create List","text":"<pre><code>lst = [i for i in range(10)]\nprint(lst)\n</code></pre> <pre><code>[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n</code></pre>"},{"location":"programming/python/python/#filter-multiples-of-3","title":"Filter multiples of 3","text":"<pre><code>multiples_of_three = filter(lambda x: x % 3 == 0, lst)\nprint(list(multiples_of_three))\n</code></pre> <pre><code>[0, 3, 6, 9]\n</code></pre>"},{"location":"programming/python/python/#reduce","title":"Reduce","text":"<p>The syntax is <code>reduce(function, sequence)</code>. The function is applied to the elements in the list in a sequential manner. Meaning if <code>lst = [1, 2, 3, 4]</code> and you have a sum function, you would arrive with <code>((1+2) + 3) + 4</code>.</p> <pre><code>from functools import reduce\nsum_all = reduce(lambda x, y: x + y, lst)\n# Here we've 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9\nprint(sum_all)\nprint(1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9)\n</code></pre> <pre><code>45\n45\n</code></pre>"},{"location":"programming/python/python/#partial","title":"Partial","text":"<p>Allows us to predefine and freeze a function's argument. Combined with lambda, it allows us to have more flexibility beyond lambda's restriction of a single line.</p> <pre><code>from functools import partial\n\ndef display_sum_three(a, b, c):\n    sum_all = a + b + c\n    print(f'Sum is {sum_all}')\n\nfixed_args_func = partial(display_sum_three, b=3, c=4)\n\n# Given fixed arguments b=3 and c=4\n# We add the new variable against the fixed arguments\nvar_int = 1\nfixed_args_func(var_int)\n\n# More advanced mapping with partial\n# Add a variable from 0 to 9 to the constants\nprint('-'*50)\n_ = list(map(fixed_args_func, list(range(10))))\n\n# How about using with lambda to modifying constants without\n# declaring your function again?\nprint('-'*50)\n_ = list(map(lambda x: fixed_args_func(x, b=2), list(range(10))))\n</code></pre> <pre><code>Sum is 8\n--------------------------------------------------\nSum is 7\nSum is 8\nSum is 9\nSum is 10\nSum is 11\nSum is 12\nSum is 13\nSum is 14\nSum is 15\nSum is 16\n--------------------------------------------------\nSum is 6\nSum is 7\nSum is 8\nSum is 9\nSum is 10\nSum is 11\nSum is 12\nSum is 13\nSum is 14\nSum is 15\n</code></pre>"},{"location":"programming/python/python/#generators","title":"Generators","text":"<ul> <li> <p>Why: <code>generators</code> are typically more memory-efficient than using simple <code>for loops</code></p> <ul> <li>Imagine wanting to sum digits 0 to 1 trillion, using a list containing those numbers and summing them would be very RAM memory-inefficient.</li> <li>Using a generator would allow you to sum one digit sequentially, staggering the RAM memory usage in steps.</li> </ul> </li> <li> <p>What: <code>generator</code> basically a function that returns an iterable object where we can iterate one bye one</p> </li> <li>Types: generator functions and generator expressions</li> <li>Dependencies: we need to install a memory profiler, so install via <code>pip install memory_profiler</code></li> </ul>"},{"location":"programming/python/python/#simple-custom-generator-function-example-sum-1-to-1000000","title":"Simple custom generator function example: sum 1 to 1,000,000","text":"<ul> <li>What: let's create a simple generator, allowing us to iterate through the digits 1 to 1,000,000 (inclusive) one by one with an increment of 1 at each step and summing them</li> <li>How: 2 step process with a <code>while</code> and a <code>yield</code></li> </ul> <pre><code># Load memory profiler\n%load_ext memory_profiler\n\n# Here we take a step from 1\ndef create_numbers(end_number):\n    current_number = 1\n\n    # Step 1: while\n    while current_number &lt;= end_number:\n        # Step 2: yield\n        yield current_number\n\n        # Add to current number\n        current_number += 1\n\n# Here we sum the digits 1 to 100 (inclusive) and time it\n%memit total = sum(create_numbers(1e6))\nprint(total)\n</code></pre> <pre><code>peak memory: 46.50 MiB, increment: 0.28 MiB\n500000500000\n</code></pre>"},{"location":"programming/python/python/#without-generator-function-sum-with-list","title":"Without generator function: sum with list","text":"<ul> <li>Say we don't use a generator, and have a list of digits 0 to 1,000,000 (inclusive) in memory then sum them. </li> <li>Notice how this is double the memory than using a generator!</li> </ul> <pre><code>%memit total = sum(list(range(int(1e6) + 1)))\nprint(total)\n</code></pre> <pre><code>peak memory: 85.14 MiB, increment: 38.38 MiB\n500000500000\n</code></pre>"},{"location":"programming/python/python/#without-generator-function-sum-with-for-loop","title":"Without generator function: sum with for loop","text":"<ul> <li>Say we don't use a generator and don't put all our numbers into a list </li> <li>Notiice how this is much better than summing a list but still worst than a generator in terms of memory?</li> </ul> <pre><code>def sum_with_loop(end_number):\n    total = 0\n    for i in range(end_number + 1):\n        i += 1\n        total += i\n\n    return total\n\n%memit total = sum_with_loop(int(1e6))\nprint(total)\n</code></pre> <pre><code>peak memory: 54.49 MiB, increment: 0.00 MiB\n500001500001\n</code></pre>"},{"location":"programming/python/python/#generator-expression","title":"Generator expression","text":"<ul> <li>Like list/dictionary expressions, we can have generator expressions too</li> <li>We can quickly create generators this way, allowing us to make computations on the fly rather than pre-compute on a whole list/array of numbers<ul> <li>This is more memory efficient</li> </ul> </li> </ul> <pre><code># Define the list\nlist_of_numbers = list(range(10))\n\n# Find square root using the list comprehension\nlist_of_results = [number ** 2 for number in list_of_numbers]\nprint(list_of_results)\n\n# Use generator expression to calculate the square root\ngenerator_of_results = (number ** 2 for number in list_of_numbers)\nprint(generator_of_results)\n\nfor idx in range(10):\n    print(next(generator_of_results))\n</code></pre> <pre><code>[0, 1, 4, 9, 16, 25, 36, 49, 64, 81]\n&lt;generator object &lt;genexpr&gt; at 0x7f08c85aa4f8&gt;\n0\n1\n4\n9\n16\n25\n36\n49\n64\n81\n</code></pre>"},{"location":"programming/python/python/#decorators","title":"Decorators","text":"<ul> <li>This allows us to to modify our original function or even entirely replace it without changing the function's code. </li> <li>It sounds mind-boggling, but a simple case I would like to illustrate here is using decorators for consistent logging (formatted print statements).</li> <li>For us to understand decorators, we'll first need to understand:<ul> <li><code>first class objects</code></li> <li><code>*args</code></li> <li><code>*kwargs</code></li> </ul> </li> </ul>"},{"location":"programming/python/python/#first-class-objects","title":"First Class Objects","text":"<pre><code>def outer():\n    def inner():\n        print('Inside inner() function.')\n\n    # This returns a function.\n    return inner\n\n# Here, we are assigning `outer()` function to the object `call_outer`.\ncall_outer = outer()\n\n# Then we call `call_outer()` \ncall_outer()\n</code></pre> <pre><code>Inside inner() function.\n</code></pre>"},{"location":"programming/python/python/#args","title":"*args","text":"<ul> <li>This is used to indicate that positional arguments should be stored in the variable args</li> <li><code>*</code> is for iterables and positional parameters</li> </ul> <pre><code># Define dummy function\ndef dummy_func(*args):\n    print(args)\n\n# * allows us to extract positional variables from an iterable when we are calling a function\ndummy_func(*range(10))\n\n# If we do not use *, this would happen\ndummy_func(range(10))\n\n# See how we can have varying arguments?\ndummy_func(*range(2))\n</code></pre> <pre><code>(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)\n(range(0, 10),)\n(0, 1)\n</code></pre>"},{"location":"programming/python/python/#kwargs","title":"**kwargs","text":"<ul> <li><code>**</code> is for dictionaries &amp; key/value pairs</li> </ul> <pre><code># New dummy function\ndef dummy_func_new(**kwargs):\n    print(kwargs)\n\n# Call function with no arguments\ndummy_func_new()\n\n# Call function with 2 arguments\ndummy_func_new(a=0, b=1)\n\n# Again, there's no limit to the number of arguments.\ndummy_func_new(a=0, b=1, c=2)\n\n# Or we can just pass the whole dictionary object if we want\nnew_dict = {'a': 0, 'b': 1, 'c': 2, 'd': 3}\ndummy_func_new(**new_dict)\n</code></pre> <pre><code>{}\n{'a': 0, 'b': 1}\n{'a': 0, 'b': 1, 'c': 2}\n{'a': 0, 'b': 1, 'c': 2, 'd': 3}\n</code></pre>"},{"location":"programming/python/python/#decorators-as-logger-and-debugging","title":"Decorators as Logger and Debugging","text":"<ul> <li>A simple way to remember the power of decorators is that the decorator (the nested function illustrated below) can<ul> <li>(1) access the passed arguments of the decorated function and</li> <li>(2) access the decorated function</li> </ul> </li> <li>Therefore this allows us to modify the decorated function without changing the decorated function</li> </ul> <pre><code># Create a nested function that will be our decorator\ndef function_inspector(func):\n    def inner(*args, **kwargs):\n        result = func(*args, **kwargs)\n        print(f'Function args: {args}')\n        print(f'Function kwargs: {kwargs}')\n        print(f'Function return result: {result}')\n        return result\n    return inner\n\n# Decorate our multiply function with our logger for easy logging\n# Of arguments pass to the function and results returned\n@function_inspector\ndef multiply_func(num_one, num_two):\n    return num_one * num_two\n\nmultiply_result = multiply_func(num_one=1, num_two=2)\n</code></pre> <pre><code>Function args: ()\nFunction kwargs: {'num_one': 1, 'num_two': 2}\nFunction return result: 2\n</code></pre>"},{"location":"programming/python/python/#dates","title":"Dates","text":""},{"location":"programming/python/python/#get-current-date","title":"Get Current Date","text":"<pre><code>import datetime\nnow = datetime.datetime.now()\nprint(now)\n</code></pre> <pre><code>2019-08-12 14:20:45.604849\n</code></pre>"},{"location":"programming/python/python/#get-clean-string-current-date","title":"Get Clean String Current Date","text":"<pre><code># YYYY-MM-DD\nnow.date().strftime('20%y-%m-%d')\n</code></pre> <pre><code>'2019-08-12'\n</code></pre>"},{"location":"programming/python/python/#count-business-days","title":"Count Business Days","text":"<pre><code># Number of business days in a month from Jan 2019 to Feb 2019\nimport numpy as np\ndays = np.busday_count('2019-01', '2019-02')\nprint(days)\n</code></pre> <pre><code>23\n</code></pre>"},{"location":"programming/python/python/#progress-bars","title":"Progress Bars","text":""},{"location":"programming/python/python/#tqdm","title":"TQDM","text":"<p>Simple progress bar via <code>pip install tqdm</code></p> <pre><code>from tqdm import tqdm\nimport time\nfor i in tqdm(range(100)):\n    time.sleep(0.1)\n    pass\n</code></pre> <pre><code>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 100/100 [00:10&lt;00:00,  9.91it/s]\n</code></pre>"},{"location":"programming/python/python/#check-paths","title":"Check Paths","text":""},{"location":"programming/python/python/#check-path-exists","title":"Check Path Exists","text":"<ul> <li>Check if directory exists</li> </ul> <pre><code>import os\ndirectory='new_dir'\nprint(os.path.exists(directory))\n\n# Magic function to list all folders\n!ls -d */\n</code></pre> <pre><code>False\nls: cannot access '*/': No such file or directory\n</code></pre>"},{"location":"programming/python/python/#check-path-exists-otherwise-create-folder","title":"Check Path Exists Otherwise Create Folder","text":"<ul> <li>Check if directory exists, otherwise make folder</li> </ul> <pre><code>if not os.path.exists(directory):\n    os.makedirs(directory)\n\n# Magic function to list all folders\n!ls -d */\n\n# Remove directory\n!rmdir new_dir\n</code></pre> <pre><code>new_dir/\n</code></pre>"},{"location":"programming/python/python/#exception-handling","title":"Exception Handling","text":""},{"location":"programming/python/python/#try-except-finally-error","title":"Try, Except, Finally: Error","text":"<ul> <li>This is very handy and often exploited to patch up (save) poorly written code</li> <li>You can use general exceptions or specific ones like <code>ValueError</code>, <code>KeyboardInterrupt</code> and <code>MemoryError</code> to name a few</li> </ul> <pre><code>value_one = 'a'\nvalue_two = 2\n\n# Try the following line of code\ntry:\n    final_sum = value_one / value_two\n    print('Code passed!')\n# If the code above fails, code nested under except will be executed\nexcept:\n    print('Code failed!')\n# This will run no matter whether the nested code in try or except is executed\nfinally:\n    print('Ran code block regardless of error or not.')\n</code></pre> <pre><code>Code failed!\nRan code block regardless of error or not.\n</code></pre>"},{"location":"programming/python/python/#try-except-finally-no-error","title":"Try, Except, Finally: No Error","text":"<ul> <li>There won't be errors because you can divide 4 with 2</li> </ul> <pre><code>value_one = 4\nvalue_two = 2\n\n# Try the following line of code\ntry:\n    final_sum = value_one / value_two\n    print('Code passed!')\n# If the code above fails, code nested under except will be executed\nexcept:\n    print('Code failed!')\n# This will run no matter whether the nested code in try or except is executed\nfinally:\n    print('Ran code block regardless of error or not.')\n</code></pre> <pre><code>Code passed!\nRan code block regardless of error or not.\n</code></pre>"},{"location":"programming/python/python/#assertion","title":"Assertion","text":"<ul> <li>This comes in handy when you want to enforce strict requirmenets of a certain value, shape, value type, or others</li> </ul> <pre><code>for i in range(10):\n    assert i &lt;= 5, 'Value is more than 5, rejected'\n    print(f'Passed assertion for value {i}')\n</code></pre> <pre><code>Passed assertion for value 0\nPassed assertion for value 1\nPassed assertion for value 2\nPassed assertion for value 3\nPassed assertion for value 4\nPassed assertion for value 5\n\n\n\n---------------------------------------------------------------------------\n\nAssertionError                            Traceback (most recent call last)\n\n&lt;ipython-input-2-d9d077e139a9&gt; in &lt;module&gt;\n      1 for i in range(10):\n----&gt; 2     assert i &lt;= 5, 'Value is more than 5, rejected'\n      3     print(f'Passed assertion for value {i}')\n\n\nAssertionError: Value is more than 5, rejected\n</code></pre>"},{"location":"programming/python/python/#asynchronous","title":"Asynchronous","text":""},{"location":"programming/python/python/#concurrency-parallelism-asynchronous","title":"Concurrency, Parallelism, Asynchronous","text":"<ul> <li>Concurrency (single CPU core): multiple threads on a single core running in sequence, only 1 thread is making progress at any point<ul> <li>Think of 1 human, packing a box then wrapping the box</li> </ul> </li> <li>Parallelism (mutliple GPU cores): multiple threads on multiple cores running in parallel, multiple threads can be making progress<ul> <li>Think of 2 humans, one packing a box, another wrapping the box</li> </ul> </li> <li>Asynchronous: concurrency but with a more dynamic system that moves amongst threads more efficiently rather than waiting for a task to finish then moving to the next task<ul> <li>Python's <code>asyncio</code> allows us to code asynchronously</li> <li>Benefits:<ul> <li>Scales better if you need to wait on a lot of processes<ul> <li>Less memory (easier in this sense) to wait on thousands of co-routines than running on thousands of threads</li> </ul> </li> <li>Good for IO bound uses like reading/saving from databases while subsequently running other computation</li> <li>Easier management than multi-thread processing like in parallel programming<ul> <li>In the sense that everything operates sequentially in the same memory space</li> </ul> </li> </ul> </li> </ul> </li> </ul>"},{"location":"programming/python/python/#asynchronous-key-components","title":"Asynchronous Key Components","text":"<ul> <li>The three main parts are (1) coroutines and subroutines, (2) event loops, and (3) future.<ul> <li>Co-routine and subroutines<ul> <li>Subroutine: the usual function</li> <li>Coroutine: this allows us to maintain states with memory of where things stopped so we can swap amongst subroutines<ul> <li><code>async</code> declares a function as a coroutine</li> <li><code>await</code> to call a coroutine</li> </ul> </li> </ul> </li> <li>Event loops</li> <li>Future</li> </ul> </li> </ul>"},{"location":"programming/python/python/#synchronous-2-function-calls","title":"Synchronous 2 Function Calls","text":"<pre><code>import timeit\ndef add_numbers(num_1, num_2):\n    print('Adding')\n    time.sleep(1)\n    return num_1 + num_2\n\ndef display_sum(num_1, num_2):\n    total_sum = add_numbers(num_1, num_2)\n    print(f'Total sum {total_sum}')\n\ndef main():\n    display_sum(2, 2)\n    display_sum(2, 2)\n\nstart = timeit.default_timer()\n\nmain()\n\nend = timeit.default_timer()\ntotal_time = end - start\n\nprint(f'Total time {total_time:.2f}s')\n</code></pre> <pre><code>Adding\nTotal sum 4\nAdding\nTotal sum 4\nTotal time 2.00s\n</code></pre>"},{"location":"programming/python/python/#parallel-2-function-calls","title":"Parallel 2 Function Calls","text":"<pre><code>from multiprocessing import Pool\nfrom functools import partial\n\nstart = timeit.default_timer()\n\npool = Pool()\nresult = pool.map(partial(display_sum, num_2=2), [2, 2]) \n\nend = timeit.default_timer()\ntotal_time = end - start\n\nprint(f'Total time {total_time:.2f}s')\n</code></pre> <pre><code>Adding\nAdding\nTotal sum 4\nTotal sum 4\nTotal time 1.08s\n</code></pre>"},{"location":"programming/python/python/#asynchronous-2-function-calls","title":"Asynchronous 2 Function Calls","text":"<p>For this use case, it'll take half the time compared to a synchronous application and slightly faster than parallel application (although not always true for parallel except in this case)</p> <pre><code>import asyncio\nimport timeit\nimport time\n\nasync def add_numbers(num_1, num_2):\n    print('Adding')\n    await asyncio.sleep(1)\n    return num_1 + num_2 \n\nasync def display_sum(num_1, num_2):\n    total_sum = await add_numbers(num_1, num_2)\n    print(f'Total sum {total_sum}')\n\nasync def main():\n    # .gather allows us to group subroutines\n    await asyncio.gather(display_sum(2, 2), \n                         display_sum(2, 2))\n\nstart = timeit.default_timer()\n\n# For .ipynb, event loop already done\nawait main()\n\n# For .py\n# asyncio.run(main())\n\nend = timeit.default_timer()\ntotal_time = end - start\n\nprint(f'Total time {total_time:.4f}s')\n</code></pre> <pre><code>Adding\nAdding\nTotal sum 4\nTotal sum 4\nTotal time 1.0021s\n</code></pre>"},{"location":"programming/r/r/","title":"R","text":""},{"location":"programming/r/r/#quick-start-basics","title":"Quick Start Basics","text":""},{"location":"programming/r/r/#variable-object","title":"Variable Object","text":"<pre><code>var_a &lt;- 2\nprint(var_a)\n</code></pre> <pre><code>[1] 2\n</code></pre> <pre><code>var_b = 3\nprint(var_b)\n</code></pre> <pre><code>[1] 3\n</code></pre>"},{"location":"programming/r/r/#check-object-type","title":"Check Object Type","text":"<pre><code># In this case, it's a numeric type\nclass(var_a)\n</code></pre> <p>'numeric'</p>"},{"location":"programming/r/r/#list-vector","title":"List / Vector","text":"<pre><code>list_a &lt;- c(0, 1, 2)\nlist_a\n</code></pre> <ol><li>0</li><li>1</li><li>2</li></ol> <pre><code># It's inclusive start till end, unlike Python which leaves the last out\nlist_b &lt;- c(0:6)\nlist_b\n</code></pre> <ol><li>0</li><li>1</li><li>2</li><li>3</li><li>4</li><li>5</li><li>6</li></ol> <pre><code># Similar to a Python dictionary\nlist_c &lt;- list(john_height=120, doe_height=150)\nlist_c\n</code></pre> $john_height 120 $doe_height 150 <pre><code># Get keys only via names()\nnames(list_c)\n</code></pre> <ol><li>'john_height'</li><li>'doe_height'</li></ol> <pre><code># Get values only via uname()\nunname(list_c)\n</code></pre> <ol> <li>120</li> <li>150</li> </ol>"},{"location":"programming/r/r/#matrix","title":"Matrix","text":"<pre><code># Create 2 vectors\nvect_a &lt;- c(1, 2)\nvect_b &lt;- c(3, 4)\n\nvect_a\nvect_b\n</code></pre> <ol><li>1</li><li>2</li></ol> <ol><li>3</li><li>4</li></ol> <pre><code># Bind as row\nmat_c &lt;- rbind(vect_a, vect_b)\nmat_c\n</code></pre> A matrix: 2 \u00d7 2 of type dbl vect_a12 vect_b34 <pre><code># Bind as column\nmat_d &lt;- cbind(vect_a, vect_b)\nmat_d\n</code></pre> A matrix: 2 \u00d7 2 of type dbl vect_avect_b 13 24 <pre><code># Create matrix\nmat_e &lt;- matrix(c(1, 2, 3, \n                 4, 5, 6), nrow=2, ncol=3)\n\nmat_e\n</code></pre> A matrix: 2 \u00d7 3 of type dbl 135 246 <pre><code># Create matrix with byrow option\nmat_g &lt;- matrix(c(1, 2, 3, 4, 5, 6), nrow=2, ncol=3, byrow=FALSE)\n\nmat_g\n</code></pre> A matrix: 2 \u00d7 3 of type dbl 135 246 <pre><code># Create same matrix with byrow option\nmat_f &lt;- matrix(c(1, 2, 3, 4, 5, 6), nrow=2, ncol=3, byrow=TRUE)\n\nmat_f\n</code></pre> A matrix: 2 \u00d7 3 of type dbl 123 456 <pre><code># Index matrix of first row and first column\nmat_f[1,1]\n# Index matrix of second row and first column\nmat_f[2,1]\n# Index matrix of second row and third column\nmat_f[2,3]\n# Select full first row\nmat_f[1,]\n# Select full first column\nmat_f[,1]\n</code></pre> <p>1</p> <p>4</p> <p>6</p> <ol><li>1</li><li>2</li><li>3</li></ol> <ol><li>1</li><li>4</li></ol> <pre><code># Find transpose of matrix: rows to cols\nt(mat_f)\n</code></pre> A matrix: 3 \u00d7 2 of type dbl 14 25 36 <pre><code># Add matrix\nmat_1 &lt;- matrix(c(1, 2, 3, 4, 5, 6), nrow=2, ncol=3)\nmat_2 &lt;- matrix(c(1, 2, 3, 4, 5, 6), nrow=2, ncol=3)\nmat_new &lt;- mat_1 + mat_2\nmat_1\nmat_2\nmat_new\n</code></pre> A matrix: 2 \u00d7 3 of type dbl 135 246 A matrix: 2 \u00d7 3 of type dbl 135 246 A matrix: 2 \u00d7 3 of type dbl 2610 4812 <pre><code># Subtract matrix\nmat_1 &lt;- matrix(c(1, 2, 3, 4, 5, 6), nrow=2, ncol=3)\nmat_2 &lt;- matrix(c(1, 2, 3, 4, 5, 6), nrow=2, ncol=3)\nmat_new &lt;- mat_1 - mat_2\nmat_1\nmat_2\nmat_new\n</code></pre> A matrix: 2 \u00d7 3 of type dbl 135 246 A matrix: 2 \u00d7 3 of type dbl 135 246 A matrix: 2 \u00d7 3 of type dbl 000 000 <pre><code># Element-wise multiplication / Hadamard product\nmat_1 &lt;- matrix(c(1, 2, 3, 4, 5, 6), nrow=2, ncol=3)\nmat_2 &lt;- matrix(c(1, 2, 3, 4, 5, 6), nrow=2, ncol=3)\nmat_new &lt;- mat_1 * mat_2\nmat_1\nmat_2\nmat_new\n</code></pre> A matrix: 2 \u00d7 3 of type dbl 135 246 A matrix: 2 \u00d7 3 of type dbl 135 246 A matrix: 2 \u00d7 3 of type dbl 1 925 41636 <pre><code># Matrix multiplication \nmat_1 &lt;- matrix(c(1, 2, 3, 4, 5, 6), nrow=2, ncol=3)\nmat_2 &lt;- matrix(c(1, 2, 3, 4, 5, 6), nrow=3, ncol=2)\nmat_new &lt;- mat_1 %*% mat_2\nmat_1\nmat_2\nmat_new\n</code></pre> A matrix: 2 \u00d7 3 of type dbl 135 246 A matrix: 3 \u00d7 2 of type dbl 14 25 36 A matrix: 2 \u00d7 2 of type dbl 2249 2864 <pre><code># Element-wise division\nmat_1 &lt;- matrix(c(1, 2, 3, 4, 5, 6), nrow=2, ncol=3)\nmat_2 &lt;- matrix(c(1, 2, 3, 4, 5, 6), nrow=2, ncol=3)\nmat_new &lt;- mat_1 / mat_2\nmat_1\nmat_2\nmat_new\n</code></pre> A matrix: 2 \u00d7 3 of type dbl 135 246 A matrix: 2 \u00d7 3 of type dbl 135 246 A matrix: 2 \u00d7 3 of type dbl 111 111 <pre><code># Scalar multiplication with matrix\nmat_1 &lt;- matrix(c(1, 2, 3, 4, 5, 6), nrow=2, ncol=3)\nmat_new &lt;- 10 * mat_1\nmat_1\nmat_new\n</code></pre> A matrix: 2 \u00d7 3 of type dbl 135 246 A matrix: 2 \u00d7 3 of type dbl 103050 204060 <pre><code># Broadcast vector of 1x3 to matrix of 2x3\nvec_1 &lt;- c(1, 2, 3)\nmat_1 &lt;- matrix(c(1, 2, 3, 4, 5, 6), nrow=2, ncol=3)\nvec_1 + mat_1\n</code></pre> A matrix: 2 \u00d7 3 of type dbl 267 459 <pre><code># Check for determinant of matrix\nmat_A &lt;- matrix(c(1, 2, 3, 4), nrow=2, ncol=2)\nmat_A\n# Here det != 0, hence inverse exist\ndet_of_mat_A = det(mat_A)\ncat(det_of_mat_A, 'is non-zero hence the inverse exist')\n</code></pre> A matrix: 2 \u00d7 2 of type dbl 13 24 <pre><code>-2 is non-zero hence the inverse exist\n</code></pre> <pre><code># Invert of Matrix: mat'\nmat_A_inv = solve(mat_A)\nmat_A_inv\n</code></pre> A matrix: 2 \u00d7 2 of type dbl -2 1.5  1-0.5 <pre><code># I = A'A = AA'\nmat_A_identity = mat_A_inv %*% mat_A\nmat_A_identity\nmat_A_inv %*% mat_A\nmat_A %*% mat_A_inv\n</code></pre> A matrix: 2 \u00d7 2 of type dbl 10 01 A matrix: 2 \u00d7 2 of type dbl 10 01 A matrix: 2 \u00d7 2 of type dbl 10 01 <pre><code># IA = AI = A\nmat_A\nmat_A_identity %*% mat_A\nmat_A %*% mat_A_identity\n</code></pre> A matrix: 2 \u00d7 2 of type dbl 13 24 A matrix: 2 \u00d7 2 of type dbl 13 24 A matrix: 2 \u00d7 2 of type dbl 13 24"},{"location":"programming/r/r/#dataframes","title":"DataFrames","text":"<pre><code>price_var &lt;- c(100, 105, 120)\ndate_var &lt;- as.Date(c('2021-01-11','2021-01-12','2021-01-13'))\ndf &lt;- data.frame(price_var, date_var)\ndf\n</code></pre> A data.frame: 3 \u00d7 2 price_vardate_var &lt;dbl&gt;&lt;date&gt; 1002021-01-11 1052021-01-12 1202021-01-13"},{"location":"programming/r/r/#for-loop","title":"For Loop","text":"<pre><code>for (i in c(0:3)) {\n    print(i)\n}\n</code></pre> <pre><code>[1] 0\n[1] 1\n[1] 2\n[1] 3\n</code></pre>"},{"location":"programming/r/r/#while-loop","title":"While Loop","text":"<pre><code># Be careful, you might end up in an infinite loop\n# if say i = -1\ni &lt;- 0\nwhile (i &lt; 3) {\n    stdout_str = paste('Yet to reach 3, i is: ', i)\n    print(stdout_str)\n    i &lt;- i + 1\n}\n\nprint('Reached 3!')\n</code></pre> <pre><code>[1] \"Yet to reach 3, i is:  0\"\n[1] \"Yet to reach 3, i is:  1\"\n[1] \"Yet to reach 3, i is:  2\"\n[1] \"Reached 3!\"\n</code></pre>"},{"location":"programming/r/r/#math","title":"Math","text":"<pre><code># Multiplication\n2 * 2\n</code></pre> <p>4</p> <pre><code># Division\n2 / 2\n</code></pre> <p>1</p> <pre><code># Addition\n2 + 2\n</code></pre> <p>4</p> <pre><code># Subtraction\n2 - 2\n</code></pre> <p>0</p> <pre><code># Exponentiation\nexp(10)\n</code></pre> <p>22026.4657948067</p> <pre><code># Inbuilt value pi\n2*pi\n</code></pre> <p>6.28318530717959</p> <pre><code># Natural log: ln\nlog(2)\n</code></pre> <p>0.693147180559945</p> <pre><code># Log with base 2\nvar_a = log(5, base=2)\nvar_a\n</code></pre> <p>2.32192809488736</p> <pre><code># Power to go back to 2\n2**var_a\n</code></pre> <p>5</p> <pre><code># Power\n2**2\n</code></pre> <p>4</p> <pre><code># Square root\nsqrt(4)\n\n# Alternative square root via power yiled same results\n4**0.5\n</code></pre> <p>2</p> <p>2</p> <pre><code># We can square root a complex number\nsqrt(-25+0i)\n\n# But we cannot square root a real number\ncat(sprintf('---------------'))\ncat(sprintf('This will fail!'))\ncat(sprintf('---------------'))\n\nsqrt(-25)\n</code></pre> <p>0+5i</p> <pre><code>---------------This will fail!---------------\n\nWarning message in sqrt(-25):\n\u201cNaNs produced\u201d\n</code></pre> <p>NaN</p> <pre><code># Print as fractions rather than decimal\nlibrary(MASS)\nfractions(1 - 0.25)\n</code></pre> <p>\u00be</p>"},{"location":"programming/r/r/#statistics","title":"Statistics","text":"<pre><code># Create list\nlist_a &lt;- c(1, 2, 3, 4, 5, 6)\n\n# Get standard deviation\ncat('standard deviation:', sd(list_a))\n\n# Get variance\ncat('\\nvariance:', var(list_a))\n\n# Get mean\ncat('\\nmedian:', mean(list_a))\n\n# Get median\ncat('\\nmean:', median(list_a))\n\n# Get summary statistics\nsummary(list_a)\n</code></pre> <pre><code>standard deviation: 1.870829\nvariance: 3.5\nmedian: 3.5\nmean: 3.5\n\n\n   Min. 1st Qu.  Median    Mean 3rd Qu.    Max. \n   1.00    2.25    3.50    3.50    4.75    6.00\n</code></pre> <pre><code># t-test\nvar_test = t.test(list_a)\nvar_test\n</code></pre> <pre><code>    One Sample t-test\n\ndata:  list_a\nt = 4.5826, df = 5, p-value = 0.005934\nalternative hypothesis: true mean is not equal to 0\n95 percent confidence interval:\n 1.536686 5.463314\nsample estimates:\nmean of x \n      3.5\n</code></pre> <pre><code># This shows what you can call if we assign it to an object\nnames(t.test(list_a))\n</code></pre> <ol><li>'statistic'</li><li>'parameter'</li><li>'p.value'</li><li>'conf.int'</li><li>'estimate'</li><li>'null.value'</li><li>'stderr'</li><li>'alternative'</li><li>'method'</li><li>'data.name'</li></ol> <pre><code># Get confidence interval\nvar_test$conf.int\n</code></pre> <ol><li>1.53668569301968</li><li>5.46331430698032</li></ol> <pre><code># Mid-point of confidence interval is the mean\nmean(var_test$conf.int)\n</code></pre> <p>3.5</p> <pre><code># Get p-value\nvar_test$p.value\n</code></pre> <p>0.00593354451759226</p> <pre><code># Plot histogram, hist() doesn't work all the time\nbarplot(table(list_a))\n</code></pre> <p></p> <pre><code># Plot dot plot\nplot(list_a)\n</code></pre> <p></p> <pre><code># Plot dot plot\n# type: p for point, l for line, b for both\n# col: cyan, blue, green, red\nplot(list_a, xlab='variable x', ylab='variable y', main='X-Y Plot', type='b', col='blue')\n</code></pre> <p></p> <pre><code>\n</code></pre> <pre><code>\n</code></pre> <pre><code>\n</code></pre>"},{"location":"programming/sympy/calculus_sympy/","title":"Calculus wth Sympy","text":"<p>In this section, we will be covering basic calculus theory and using sympy to solve the related equations.</p>"},{"location":"programming/sympy/calculus_sympy/#calculus-differentiation-theory","title":"Calculus: Differentiation (Theory)","text":""},{"location":"programming/sympy/calculus_sympy/#basic-rules","title":"Basic Rules","text":"\\[ f(x) = x^n \\\\ f\\prime(x) = nx^{x-1} \\] \\[ f(x) = c \\\\ f\\prime(x) = 0 \\] \\[ f(x) = \\cos(x) \\\\ f\\prime(x) = -\\sin(x) \\] \\[ f(x) = \\cosh(x) \\\\ f\\prime(x) = \\sinh(x) \\] \\[ f(x) = \\sin(x) \\\\ f\\prime(x) = \\cos(x) \\] \\[ f(x) = \\sinh(x) \\\\ f\\prime(x) = \\cosh(x) \\] \\[ f(x) = \\ln (x) \\\\ f\\prime(x) = \\frac{1}{x} \\]"},{"location":"programming/sympy/calculus_sympy/#sum-difference-rule","title":"Sum Difference Rule","text":"<p>Given</p> \\[ f(x) = g(x) \\pm h(x) \\] <p>Then</p> \\[ f\\prime(x) = g\\prime(x) \\pm h\\prime(x) \\]"},{"location":"programming/sympy/calculus_sympy/#product-rule","title":"Product Rule","text":"<p>Given a function with two parts</p> \\[ f(x) = g(x) \\cdot h(x) \\] <p>First derivative</p> \\[ f\\prime(x) = g\\prime(x) \\cdot h(x) + g(x) \\cdot h\\prime(x) \\] <p>Given three parts, we always use the product rule to keep it to 2 parts</p> \\[ f(x) = g(x) \\cdot h(x) \\cdot i(x) \\] \\[ f'(x) = \\big[g(x) \\cdot h(x)\\big]\\prime \\cdot i(x) + \\big[g(x) \\cdot h(x) \\big] \\cdot i\\prime(x) \\]"},{"location":"programming/sympy/calculus_sympy/#quotient-rule","title":"Quotient Rule","text":"<p>Given</p> \\[ f(x) = \\frac{g(x)}{h(x)} \\] <p>Then</p> \\[ f\\prime(x) = \\frac{g\\prime(x) \\cdot h(x) - g(x) \\cdot h\\prime(x)}{h^2(x)} \\]"},{"location":"programming/sympy/calculus_sympy/#logarithmic-rule","title":"Logarithmic Rule","text":"<p>Given</p> \\[f(x) = ln(g(x))\\] <p>Then</p> \\[ f\\prime(x) = \\frac{g\\prime(x)}{g(x)} \\]"},{"location":"programming/sympy/calculus_sympy/#exponential-rule","title":"Exponential Rule","text":"<p>Given</p> \\[ f(x) = \\exp \\big(g(x)\\big) \\] <p>Then</p> \\[ f\\prime(x) = g\\prime(x) \\exp \\big(g(x)\\big) \\]"},{"location":"programming/sympy/calculus_sympy/#partial-derivative","title":"Partial Derivative","text":"<p>Given</p> \\[ f(x) = g(x_{i}, x_{i+1}, \\dots, x_n) \\] <p>When computing the partial derivative of \\(x_i\\) with respect to (w.r.t.) the multivariable function \\(f(x_i \\dots x_n)\\), treat the rest of the variables as constant.</p>"},{"location":"programming/sympy/calculus_sympy/#calculus-integration-theory","title":"Calculus: Integration (Theory)","text":""},{"location":"programming/sympy/calculus_sympy/#basic-rules_1","title":"Basic Rules","text":"<p>Given function \\(f(x)\\) and integral of that function \\(F(x)\\), then basic rules include</p> \\[ f(x) = x^n \\\\ F(x) = \\frac{x^{n+1}}{n+1} + c \\] \\[ f(x) = 0 \\\\ F(x) = c \\] \\[ f(x) = e^x \\\\ F(x) = e^x + c \\] \\[ f(x) = a^x \\\\ F(x) = \\frac{a^x}{\\ln a} + c \\] \\[ f(x) = \\sin(x) \\\\ F(x) = -\\cos(x) + c \\] \\[ f(x) = \\sinh(x) \\\\ F(x) = \\cosh(x) + c \\] \\[ f(x) = \\cos(x) \\\\ F(x) = \\sin(x) + c \\] \\[ f(x) = \\cosh(x) \\\\ F(x) = \\sinh(x) + c \\] \\[ f(x) = \\frac{1}{x} \\\\ F(x) = \\ln\\left|x\\right| + c \\] \\[ f(x) = m \\\\ F(x) = mx + c \\] \\[ f(x) = \\frac{1}{x^2 + a^2} \\\\ F(x) = (\\frac{1}{a}) \\cdot \\arctan(\\frac{x}{a}) + c \\] \\[ f(x) = \\frac{1}{\\sqrt{a^2 - x^2}} \\\\ F(x) = \\arcsin(\\frac{x}{a}) + c \\]"},{"location":"programming/sympy/calculus_sympy/#sum-difference-rule_1","title":"Sum Difference Rule","text":"<p>Given</p> \\[ f(x) = g(x) \\pm h(x) \\] <p>Then</p> \\[ F(x) = \\int \\big( g(x) \\pm h(x) \\big) \\, dx\\]"},{"location":"programming/sympy/calculus_sympy/#integration-by-parts","title":"Integration by Parts","text":"\\[ F(x) = \\int u\\ v\\prime \\ dx = u\\ v - \\int v\\ u\\prime \\ dx \\]"},{"location":"programming/sympy/calculus_sympy/#definite-integral-rules","title":"Definite Integral Rules","text":""},{"location":"programming/sympy/calculus_sympy/#zero-integral-when-not-moving-integration-point","title":"Zero integral when not moving integration point","text":"\\[ \\int _a ^a f(x) \\ dx = 0 \\]"},{"location":"programming/sympy/calculus_sympy/#definite-integrals-interval-switching-becomes-negative","title":"Definite integral's interval switching becomes negative","text":"\\[ \\int _a ^b f(x) \\ dx = - \\int _b ^a f(x) \\ dx  \\]"},{"location":"programming/sympy/calculus_sympy/#definite-integral-decomposed-into-parts","title":"Definite integral decomposed into parts","text":"\\[ \\int _a ^b f(x) \\ dx = \\int _a ^c f(x) \\ dx - \\int _b ^c f(x) \\ dx \\]"},{"location":"programming/sympy/calculus_sympy/#substitution-method-for-solving-definite-integrals","title":"Substitution method for solving definite integrals","text":""},{"location":"programming/sympy/calculus_sympy/#substitution-example-1","title":"Substitution Example 1","text":"<p>Given $$ f(x) = x \\cos(x^2 + 1) $$</p> <p>Then definite integral from -1 to 1 is  $$ F(x)_{-1}^1 = \\int _{-1} ^ 1 x \\cos(x^2 + 1)  dx $$</p> <p>Modifying the equation to make \\(du\\) and \\(u\\) substitution method results in $$ F(x)_{-1}^1 = \\frac{1}{2} \\int _{-1} ^ 1 2x \\cos(x^2 + 1)  dx $$</p> <p>Let \\(u\\) be $$ u = x^2 +1  $$</p> <p>Let \\(du\\) be $$ du = 2x $$</p> <p>New limits given -1 and 1 $$ u = (-1)^2 + 1 = 2 $$ $$ u = (1)^2 + 1 = 2 $$</p> <p>Then by zero integral rule $$ F(x)_{-1}^1 = \\int _2 ^2 cos(u) du = 0 $$</p>"},{"location":"programming/sympy/calculus_sympy/#substitution-example-2","title":"Substitution Example 2","text":"<p>Given $$ h(x) = x(x+3)^{\\frac{1}{2}} $$</p> <p>Then definite integral from -1 to 1 is $$ H(x)_{-1}^1 = \\int _{-1} ^ 1 x(x+3)^{\\frac{1}{2}} $$</p> <p>Let \\(u\\) be $$ u = x + 3 $$ $$ x = u - 3 $$</p> <p>Let \\(du\\) be $$ du = 1 $$</p> <p>New limits given -1 and 1 $$ u = -1 + 3 = 2 $$ $$ u = 1 + 3 = 4 $$</p> <p>Then</p> \\[ H(x)_{-1}^1 = \\int _{2} ^ 4 (u-3)(u)^{\\frac{1}{2}} \\ du \\] \\[ H(x)_{-1}^1 = \\int _{2} ^ 4 u^{\\frac{3}{2}} - 3u^{\\frac{1}{2}} \\ du \\] \\[ H(x)_{-1}^1 = \\big[ \\frac{u^{\\frac{5}{2}}}{\\frac{5}{2}} - \\frac{3u^{\\frac{3}{2}}}{\\frac{3}{2}} \\big]_2 ^4 \\] \\[ H(x)_{-1}^1 = \\frac{2}{5} [-8 + 6\\sqrt(2)] \\]"},{"location":"programming/sympy/calculus_sympy/#substitution-example-3","title":"Substitution Example 3","text":"<p>Keep on substituting!</p> <p>You can keep running the substitution rule multiple times, where you can go from \\(u\\) to \\(z\\) to etc. to solve the definite integral. It is important to take note that your limits change every time you run the substitution rule once!</p>"},{"location":"programming/sympy/calculus_sympy/#multiple-integrals","title":"Multiple Integrals","text":"<p>This is used for functions with multiple variables. Hence, instead of the usual single variable where the definite integral represents the area under the curve, multiple integrals calculate hypervolumes with multiple dimensional functions.</p> <p>The key is in integrating step by step, when integrating with respect to each variable, the rest of the variables act as constants.</p>"},{"location":"programming/sympy/calculus_sympy/#double-integral","title":"Double Integral","text":"\\[V = \\int _{x=a}^{x=b} \\int _{y=c}^{y=d} f(x,y) \\ dy \\ dx\\]"},{"location":"programming/sympy/calculus_sympy/#triple-integral","title":"Triple Integral","text":"\\[V = \\int _{x=a}^{x=b} \\int _{y=c}^{y=d} \\int _{z=e}^{z=f} f(x,y,z) \\ dz \\ dy \\ dx\\]"},{"location":"programming/sympy/calculus_sympy/#calculus-taylor-expansion-theory","title":"Calculus: Taylor Expansion (Theory)","text":""},{"location":"programming/sympy/calculus_sympy/#taylor-expansion-single-variable","title":"Taylor Expansion: Single Variable","text":"<p>Given a function that has continuous derivatives up to \\((n+1)\\) order, the function can be expanded with the following equation:</p> \\[ F(x) = \\sum^{\\infty}_{n=0} \\frac{F^{(n)}(a)}{n!}\\ (x-a)^n \\] \\[ F(x) = F(a) + F\\prime (x)(x-a) + \\frac{F\\prime\\prime(a)}{2!}(x-a)^2 + \\cdots + \\frac{F(n)(a)}{n!} + R_n(x) \\] <p>If not expanded till \\(\\infty\\) then there will be a remainder of \\(R_n(x)\\)</p>"},{"location":"programming/sympy/calculus_sympy/#taylor-expansion-multiple-variables","title":"Taylor Expansion: Multiple Variables","text":"<p>In the case of 2 variables \\((x, y)\\), we can expand the multivariate equation with the generalized Taylor expansion equation:</p> \\[ F(x, y) = \\\\ F(a, b) + \\big[ \\frac{\\partial}{\\partial x} F(x, y)_{a,b} (x-a) + \\frac{\\partial}{\\partial y} F(x, y)_{a,b} ) (y - b)  \\big] \\\\ + \\frac{1}{2!} \\big[ \\frac{\\partial ^2}{\\partial x^2} F(x, y)_{a,b} (x-a)^2 + \\frac{\\partial ^2}{\\partial y^2} F(x, y)_{a,b}(y-b)^2 \\big] \\\\ + \\frac{\\partial ^2}{\\partial x \\partial y} F(x, y)_{a,b} (x-a)(y-b) + R_2 (x,y) \\]"},{"location":"programming/sympy/calculus_sympy/#maclaurin-expansion","title":"Maclaurin Expansion","text":"<p>It is simply a Taylor expansion about the point 0.</p>"}]}
\ No newline at end of file
diff --git a/sitemap.xml b/sitemap.xml
index a434f5ae1..53e4efecf 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -2,302 +2,317 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
     <url>
          <loc>https://www.deeplearningwizard.com/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/about/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/consultancy/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/home/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/pipeline/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/review/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/data_engineering/nosql/cassandra/intro/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/data_engineering/nosql/cassandra/setting_up_cluster/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/course_progression/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/intro/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/readings/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/boosting_models_pytorch/derivative_gradient_jacobian/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/boosting_models_pytorch/forwardpropagation_backpropagation_gradientdescent/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/boosting_models_pytorch/lr_scheduling/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/boosting_models_pytorch/optimizers/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/deep_reinforcement_learning_pytorch/bellman_mdp/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/deep_reinforcement_learning_pytorch/dynamic_programming_frozenlake/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/deep_reinforcement_learning_pytorch/rl_notes/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/deep_reinforcement_learning_pytorch/supervised_to_rl/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/fromscratch/fromscratch_cnn/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/fromscratch/fromscratch_logistic_regression/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_autoencoder/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_convolutional_neuralnetwork/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_fc_overcomplete_ae/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_feedforward_neuralnetwork/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_gradients/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_linear_regression/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_logistic_regression/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_matrices/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_recurrent_neuralnetwork/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/deep_learning/production_pytorch/speed_optimization_basics_numba/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
+         <changefreq>daily</changefreq>
+    </url>
+    <url>
+         <loc>https://www.deeplearningwizard.com/language_model/intro/</loc>
+         <lastmod>2024-03-10</lastmod>
+         <changefreq>daily</changefreq>
+    </url>
+    <url>
+         <loc>https://www.deeplearningwizard.com/language_model/containers/hpc_containers_apptainer/</loc>
+         <lastmod>2024-03-10</lastmod>
+         <changefreq>daily</changefreq>
+    </url>
+    <url>
+         <loc>https://www.deeplearningwizard.com/language_model/llm/llm_intro_hyperparameter_tuning/</loc>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/machine_learning/intro/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/machine_learning/gpu/gpu_fractional_differencing/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/machine_learning/gpu/rapids_cudf/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/news/ammi_facebook_google_recap_2018_11_21/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/news/dbs_gpu_rapids_nvidia_ensemble_frac_diff/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/news/deep_learning_wizard_1y_2018_06_01/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/news/deep_learning_wizard_nvidia_inception_2018_05_01/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/news/defence_and_science_technology_agency_dsta_nvidia_talk_2016_06/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/news/detect_waterbone_debris_ai_for_social_good_icml_2019_06/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/news/facebook_pytorch_devcon_recap_2018_10_02/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/news/facebook_pytorch_developer_conference_2018_09_05/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/news/it_youth_leader_2019_03_11/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/news/nanjing_next_nus_tsinghua_ai_finance_healthcare_2018_11_01/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/news/news/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/news/nvidia_nus_mit_datathon_2018_07_05/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/news/nvidia_self_driving_cars_talk_2017_06_21/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/ood/intro/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/programming/intro/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/programming/bash/bash/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/programming/cpp/cpp/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/programming/electron/electron/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/programming/javascript/javascript/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/programming/numpycupy/linalg_numpy_cupy/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/programming/plotting/intro/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/programming/python/python/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/programming/r/r/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://www.deeplearningwizard.com/programming/sympy/calculus_sympy/</loc>
-         <lastmod>2023-11-06</lastmod>
+         <lastmod>2024-03-10</lastmod>
          <changefreq>daily</changefreq>
     </url>
 </urlset>
\ No newline at end of file
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index aed5db20a8b466637f5bbfb44c494d2ddcbd9ebe..80ac86da58710477298ceac34bdd209bfcd2aafa 100644
GIT binary patch
delta 1119
zcmV-l1fcu22<!+4ABzYGn0W1x2OfXZ+=^Z2tzXbS560w3;x0upL(-1&^&LvK*J;p~
z0tG}Fkj0zmbI9SDA?2s9-v@hlB#y?r=gsb6ySXE$yf&_T-u(6FANQX(U-$2xGTTV$
zP8z%Tyy;U4zc<b4bb5#;(SRX7c<!30ddbA*T-a<MzHIjI?&MQCm{Zf|$$EdaFTKQb
zH<~C|LU!`IC-VbYKPW#mPfclf`y9qJc)fqzKK^#UeSg2(N|SThO*7S_>!@Y=w%2Iv
z_|)8#%z&CJqW_2u4?d?wBUIkSY!f;)E*fX=2U;}1U;<g=mJO2WXy~+Ph#C_DxlV?X
zSgIQ5<2TzVj1iq?Y>Z3n>!yDoGMS_tasXv>OvKCPBgJqj7sbdN2?G0$DHfkl5%>y>
z=O?^OSln!^2=c-EI9azKAsZo#$+POE6`La_b0p|kSdmNcg32EZx)soCJ)h(WSr!31
zq3InQP`%#fFSIp9CASrjwhSen)7h3*_4Dx|nZf*!P<I7nPGq`1fpLE(8MN~BWQ3^F
zv<xNLW~WM`CbCnG$v9bWiiRR0m%1zQ16ezugF%OtG1H^N!Km5!;xLrDmi-TM7GG&)
zWZ8oAiPmPFF$dzKiKHQUU|SJAF|{Eo##H>>TN;#eD_~)LXGBPqOkjnPZQ!Ep3hmX<
z&Ue6^{B&MZ)i&eb`6z$yj^5_txun)nX0(pd$@A+f__YfBkysv)UJ?P)I%pB;bCA5k
zNxQ`MaTRRJK^F4bgmA3_Zw1{7m(+lXjiPm|C?=JeOF&rfM0uU3{o-3F8${FLylK9i
zc|@ZkxI7$i(d4Xjn6aaI*IdED;6DIM6qHb8nq$)k1#elv5axemO>|_(UM`sC0s2w)
zNC9PwD*+Hg%5LfBO9QH2#=NGJ=3gLpA(so<NyVn<TAQ{dku6o-_!lya!RQDo>vqxL
z9i3uRj&q<zMF;O+q4U18RNMvzc--zj!)^zUyXDi+hX}=p&SUS6Mw{|BEYffgLdCWw
ztEzy>t^0$y7IuH*JR94Ou-#S4;XEhCDGHO*o*8|pgi$MUB)ej#Au2Pa6RF;L3GYV`
zkK)t_IwrjEOlV&VMVYaWHSiNQ2~H@E@8CtzG#%K8R~K>c4ACnix-9ab!KlF&VqdBt
zc9phENUZ57QOi#fw}tIvolMAj30X=QHv3CDFDjXv56gdNWD<-%r@lyq78<iry%(Re
zeC!V0OA$9m-(_=^Bcv$VKI61xj8w~H^n^C`3Yo6Xv<9)8zLt-2*4Z-lgIH4e`%CJt
zZsk)IE8xr_l5Ih=ITni&bBR~)@u|4{g$H2b6e`$v+`PQ34DA79KY!JNScNcMwcsTy
z!4G@)mZ(n!`3oK~DmEeACTl_O@(@Oq!`N8yn6`td0H%^!I0Wmbzg1QyDiTDBv%K7D
lLCp<UgEfv~%Z);-vXvNR&i?P<r(2}7{~HWP7<)i500579E-C;3

delta 1065
zcmV+^1lIfP2(}0ZABzYGCR<362OfWeY?5NvdFvOn&x0{JlDJEe%#f6${rV2&OB}R)
zDNsO^0a?6BK1ZB6Go<|d?Z;s6M&f9^d)e$BwwpV0%4_4gm(8Dl{C@vs^KJj(IkSzF
z?qslwFPlE4@LSUykH?2-5)ByQgXgY^s+US^&W+9X;p=Aq;Z8o~2B*^edA5IE?aN)_
z`8Jv;SVDI4yGQemtRIvgn&+kryuA)%8ob^=ZXZA1?{@dwCmC{XyBVf>bRD%!-}f4g
zot~SUmJ^`nis(OL!-LOh(Fm1yG24VrO^e3a`+*h>FqlBrxMhQ68Vwy64N+r4AlJ#G
zB$lqm`S{&73S&g485`pg`?`NAh)gDFha5oJ924=f`A9Qd+C?!kM}ok<V~QmvR0O^P
z<M{<I6P7R=D}sFRKF-!HNXSMAQ}V2O8O3JAWJZFHg%!C3uc-XNpj!dG*7H%W5Vr`}
z2~F?dfa>kGexa=?Dmkryv}GvyoX)n4s-KS!$qeRQQr#7hIg;u61jc`vWYEghvk{_7
zvl~jW%}$j<O=Ra9lX2p2iiRR0m%c0U16ezugF%OtG1HU7!KhR4#bGFYE&F$J7T;)P
zWZ8oAiPmPFF$dz&MADEvu&s!mnA#8(V=D3PXBm`pD_~)LXGBPqOkjnXZQ!Ep3hmX<
z&S$`!{JdXN)iUGX`6z$yMsIV8T+-_(Gg?RK==p6G{8|NmB$i9$E{OnX9kht_IZ0mO
zq+N3RxC%C9BMW(LLbz6ew}NhkE;V3cqi7u-#iTNGNeJtmD6jLpUi_KL2GMjlFPblV
z9?_@>E)NG>G`Rs)FDh5l*5KbDcOjQUmVl_v>5w*UOJZkg=^KAC|44>07#%@n-7Xru
zqhoB!`WsqQbnyNSI`2D6#m_Oo<97E2b~||7EuV%yL?}jde)euO+LYH9k%og1Dz-IS
zRRv6L^$JeCu$xY^v3-K=u2K%?IWbOAm{s<~=u;((T9G4J#yAa8nYo-u_0CH^HUW0h
zGJ%c>FFX_4w?cnWX6&g3e!?cf5fkyji=t^du!(>o4xS-;Wki=n9yAy=*h1`U6~wO6
zcFAEhjk5OsMdG%weXNrSSs#eYX2L06(%GqGZayuakx4N5ocbaaT4<b%>Z8P*<zsi~
zUW>Rn`XSkHj*y~M`;4>880nVD=m~A=6*66eX$@jGe=UC><w<ADvmeBg%KsHoe+?_2
ztMGs`he);s&1NhXCFT;Z(c^Oo`3q0LBq&s{?znl|R2f=XjrOOnS`e!c=A#z8WF`4w
z?|v3)LH>#(M#Uzi+hQ%~T^_=ua+n$`5z}@sH^AId3x{C+{6DzLL`9NF36_^rEvUKS
jYOuyhY`GaxXjQh7qnxV$JNWq&DeeCP-craX@h$)Wwv-Jf