NeMo Multimodal Docs and Tests Initial PR

Jenkinsfile

@@ -115,6 +115,256 @@ pipeline {
         sh 'CUDA_VISIBLE_DEVICES="" NEMO_NUMBA_MINVER=0.53 pytest -m "not pleasefixme" --cpu --with_downloads --relax_numba_compat'
       }
     }
+//
+//     stage('L2: Multimodal Imagen Train') {
+//       when {
+//         anyOf {
+//           branch 'main'
+//           changeRequest target: 'main'
+//         }
+//       }
+//       failFast true
+//       steps {
+//         sh "rm -rf /home/TestData/multimodal/imagen_train"
+//         sh "pip install webdataset==0.2.48"
+//         sh "python examples/multimodal/text_to_image/imagen/imagen_training.py \
+//         trainer.precision=16 \
+//         trainer.num_nodes=1 \
+//         trainer.devices=1 \
+//         ++exp_manager.max_time_per_run=00:00:03:00 \
+//         trainer.max_steps=20 \
+//         model.micro_batch_size=1 \
+//         model.global_batch_size=1 \
+//         model.data.synthetic_data=True \
+//         exp_manager.exp_dir=/home/TestData/multimodal/imagen_train \
+//         model.inductor=False \
+//         model.unet.flash_attention=False \
+//         "
+//         sh "pip install 'webdataset>=0.1.48,<=0.1.62'"
+//         sh "rm -rf /home/TestData/multimodal/imagen_train"
+//       }
+//     }
+//
+//     stage('L2: Multimodal Stable Diffusion Train') {
+//       when {
+//         anyOf {
+//           branch 'main'
+//           changeRequest target: 'main'
+//         }
+//       }
+//       failFast true
+//       steps {
+//         sh "rm -rf /home/TestData/multimodal/stable_diffusion_train"
+//         sh "pip install webdataset==0.2.48"
+//         sh "python examples/multimodal/text_to_image/stable_diffusion/sd_train.py \
+//             trainer.precision=16 \
+//             trainer.num_nodes=1 \
+//             trainer.devices=1 \
+//             ++exp_manager.max_time_per_run=00:00:03:00 \
+//             trainer.max_steps=20 \
+//             model.micro_batch_size=1 \
+//             model.global_batch_size=1 \
+//             model.data.synthetic_data=True \
+//             exp_manager.exp_dir=/home/TestData/multimodal/stable_diffusion_train \
+//             model.inductor=False \
+//             model.cond_stage_config._target_=nemo.collections.multimodal.modules.stable_diffusion.encoders.modules.FrozenCLIPEmbedder \
+//             ++model.cond_stage_config.version=openai/clip-vit-large-patch14 \
+//             ++model.cond_stage_config.max_length=77 \
+//             ~model.cond_stage_config.restore_from_path \
+//             ~model.cond_stage_config.freeze \
+//             ~model.cond_stage_config.layer \
+//             model.unet_config.from_pretrained=null \
+//             model.first_stage_config.from_pretrained=null \
+//             model.unet_config.use_flash_attention=False \
+//             "
+//         sh "pip install 'webdataset>=0.1.48,<=0.1.62'"
+//         sh "rm -rf /home/TestData/multimodal/stable_diffusion_train"
+//       }
+//     }
+//     stage('L2: Multimodal ControlNet Train') {
+//       when {
+//         anyOf {
+//           branch 'main'
+//           changeRequest target: 'main'
+//         }
+//       }
+//       failFast true
+//       steps {
+//         sh "rm -rf /home/TestData/multimodal/controlnet_train"
+//         sh "pip install webdataset==0.2.48"
+//         sh "python examples/multimodal/text_to_image/controlnet/controlnet_train.py \
+//             trainer.precision=16 \
+//             trainer.num_nodes=1 \
+//             trainer.devices=1 \
+//             ++exp_manager.max_time_per_run=00:00:03:00 \
+//             trainer.max_steps=20 \
+//             model.micro_batch_size=1 \
+//             model.global_batch_size=1 \
+//             model.data.synthetic_data=True \
+//             exp_manager.exp_dir=/home/TestData/multimodal/controlnet_train \
+//             model.inductor=False \
+//             model.image_logger.max_images=0 \
+//             model.control_stage_config.params.from_pretrained_unet=null \
+//             model.unet_config.from_pretrained=null \
+//             model.first_stage_config.from_pretrained=null \
+//             model.unet_config.use_flash_attention=False \
+//             "
+//         sh "pip install 'webdataset>=0.1.48,<=0.1.62'"
+//         sh "rm -rf /home/TestData/multimodal/controlnet_train"
+//       }
+//     }
+//     stage('L2: Multimodal DreamBooth Train') {
+//       when {
+//         anyOf {
+//           branch 'main'
+//           changeRequest target: 'main'
+//         }
+//       }
+//       failFast true
+//       steps {
+//         sh "rm -rf /home/TestData/multimodal/dreambooth_train"
+//         sh "pip install webdataset==0.2.48"
+//         sh "python examples/multimodal/text_to_image/dreambooth/dreambooth.py \
+//             trainer.precision=16 \
+//             trainer.num_nodes=1 \
+//             trainer.devices=1 \
+//             ++exp_manager.max_time_per_run=00:00:03:00 \
+//             trainer.max_steps=20 \
+//             model.micro_batch_size=1 \
+//             model.global_batch_size=1 \
+//             exp_manager.exp_dir=/home/TestData/multimodal/dreambooth_train \
+//             model.inductor=False \
+//             model.cond_stage_config._target_=nemo.collections.multimodal.modules.stable_diffusion.encoders.modules.FrozenCLIPEmbedder \
+//             ++model.cond_stage_config.version=openai/clip-vit-large-patch14 \
+//             ++model.cond_stage_config.max_length=77 \
+//             ~model.cond_stage_config.restore_from_path \
+//             ~model.cond_stage_config.freeze \
+//             ~model.cond_stage_config.layer \
+//             model.unet_config.from_pretrained=null \
+//             model.first_stage_config.from_pretrained=null \
+//             model.data.instance_dir=/home/TestData/multimodal/tiny-dreambooth \
+//             model.unet_config.use_flash_attention=False \
+//             "
+//         sh "pip install 'webdataset>=0.1.48,<=0.1.62'"
+//         sh "rm -rf /home/TestData/multimodal/dreambooth_train"
+//       }
+//     }
+//     stage('L2: Vision ViT Pretrain TP=1') {
+//       when {
+//         anyOf {
+//           branch 'main'
+//           changeRequest target: 'main'
+//         }
+//       }
+//       failFast true
+//       steps {
+//         sh "rm -rf /home/TestData/vision/vit_pretrain_tp1"
+//         sh "pip install webdataset==0.2.48"
+//         sh "python examples/vision/vision_transformer/megatron_vit_classification_pretrain.py \
+//             trainer.precision=16 \
+//             model.megatron_amp_O2=False \
+//             trainer.num_nodes=1 \
+//             trainer.devices=1 \
+//             trainer.val_check_interval=5 \
+//             ++exp_manager.max_time_per_run=00:00:03:00 \
+//             trainer.max_steps=20 \
+//             model.micro_batch_size=2 \
+//             model.global_batch_size=4 \
+//             model.tensor_model_parallel_size=1 \
+//             model.pipeline_model_parallel_size=1 \
+//             model.data.num_workers=0 \
+//             exp_manager.create_checkpoint_callback=False \
+//             model.data.data_path=[/home/TestData/multimodal/tiny-imagenet/train,/home/TestData/multimodal/tiny-imagenet/val] \
+//             exp_manager.exp_dir=/home/TestData/vision/vit_pretrain_tp1 "
+//         sh "pip install 'webdataset>=0.1.48,<=0.1.62'"
+//         sh "rm -rf /home/TestData/vision/vit_pretrain_tp1"
+//       }
+//     }
+//
+//     stage('L2: Multimodal CLIP Pretrain TP=1') {
+//       when {
+//         anyOf {
+//           branch 'main'
+//           changeRequest target: 'main'
+//         }
+//       }
+//       failFast true
+//       steps {
+//         sh "rm -rf /home/TestData/multimodal/clip_pretrain_tp1"
+//         sh "pip install webdataset==0.2.48"
+//         sh "python examples/multimodal/vision_language_foundation/clip/megatron_clip_pretrain.py  \
+//             trainer.precision=16 \
+//             model.megatron_amp_O2=False \
+//             trainer.num_nodes=1 \
+//             trainer.devices=1 \
+//             trainer.val_check_interval=10 \
+//             ++exp_manager.max_time_per_run=00:00:03:00 \
+//             trainer.max_steps=20 \
+//             model.micro_batch_size=1 \
+//             model.global_batch_size=1 \
+//             model.tensor_model_parallel_size=1 \
+//             model.pipeline_model_parallel_size=1 \
+//             exp_manager.create_checkpoint_callback=False \
+//             model.data.num_workers=0 \
+//             model.vision.num_layers=2 \
+//             model.text.num_layers=2 \
+//             model.vision.patch_dim=32 \
+//             model.vision.encoder_seq_length=49 \
+//             model.vision.class_token_length=7 \
+//             model.data.train.dataset_path=[/home/TestData/multimodal/tiny-clip/00000.tar] \
+//             model.data.validation.dataset_path=[/home/TestData/multimodal/tiny-clip/00000.tar] \
+//             model.data.webdataset.local_root_path=/ \
+//             exp_manager.exp_dir=/home/TestData/multimodal/clip_pretrain_tp1 "
+//         sh "pip install 'webdataset>=0.1.48,<=0.1.62'"
+//         sh "rm -rf /home/TestData/multimodal/clip_pretrain_tp1"
+//       }
+//     }
+//
+//     stage('L2: Multimodal NeVA Pretrain TP=1') {
+//       when {
+//         anyOf {
+//           branch 'main'
+//           changeRequest target: 'main'
+//         }
+//       }
+//       failFast true
+//       steps {
+//         sh "rm -rf /home/TestData/multimodal/neva_pretrain_tp1"
+//         sh "pip install webdataset==0.2.48"
+//         sh "python examples/multimodal/multimodal_llm/neva/neva_pretrain.py \
+//             trainer.precision=bf16 \
+//             model.megatron_amp_O2=False \
+//             trainer.num_nodes=1 \
+//             trainer.devices=1 \
+//             trainer.val_check_interval=10 \
+//             trainer.limit_val_batches=5 \
+//             trainer.log_every_n_steps=1 \
+//             ++exp_manager.max_time_per_run=00:00:03:00 \
+//             trainer.max_steps=20 \
+//             model.micro_batch_size=2 \
+//             model.global_batch_size=4 \
+//             model.tensor_model_parallel_size=1 \
+//             model.pipeline_model_parallel_size=1 \
+//             exp_manager.create_checkpoint_callback=False \
+//             model.data.data_path=/home/TestData/multimodal/tiny-neva/dummy.json \
+//             model.data.image_folder=/home/TestData/multimodal/tiny-neva/images \
+//             model.tokenizer.library=sentencepiece \
+//             model.tokenizer.model=/home/TestData/multimodal/tiny-neva/tokenizer_add_special.model \
+//             model.num_layers=2 \
+//             model.hidden_size=5120 \
+//             model.ffn_hidden_size=13824 \
+//             model.num_attention_heads=40 \
+//             model.normalization=rmsnorm \
+//             model.data.num_workers=0 \
+//             model.data.conv_template=llama_2 \
+//             model.mm_cfg.vision_encoder.from_pretrained='openai/clip-vit-large-patch14' \
+//             model.mm_cfg.llm.from_pretrained=null \
+//             model.use_flash_attention=false \
+//             exp_manager.exp_dir=/home/TestData/multimodal/neva_pretrain_tp1 "
+//         sh "pip install 'webdataset>=0.1.48,<=0.1.62'"
+//         sh "rm -rf /home/TestData/multimodal/neva_pretrain_tp1"
+//       }
+//     }
 
     // TODO: this requires TE >= v0.11 which is not available in 23.06.
     //        please uncomment this test once mcore CI is ready.
@@ -4815,6 +5065,7 @@ assert_frame_equal(training_curve, gt_curve, rtol=1e-3, atol=1e-3)"'''
         }
       }
     }
+
     stage('L2: TTS Fast dev runs 1') {
       when {
         anyOf {
@@ -4960,7 +5211,27 @@ assert_frame_equal(training_curve, gt_curve, rtol=1e-3, atol=1e-3)"'''
         }
       }
     }
-
+    stage('L2: NeRF') {
+      when {
+        anyOf {
+          branch 'r1.21.0'
+          changeRequest target: 'r1.21.0'
+        }
+      }
+      parallel {
+        stage('DreamFusion') {
+          steps {
+            sh 'python examples/multimodal/text_to_image/nerf/main.py \
+            trainer.num_nodes=1 \
+            trainer.devices="[0]" \
+            trainer.max_steps=1000 \
+            model.prompt="a DSLR photo of a delicious hamburger" \
+            exp_manager.exp_dir=examples/multimodal/text_to_image/nerf/dreamfusion_results'
+            sh 'rm -rf examples/multimodal/text_to_image/nerf/dreamfusion_results'
+          }
+        }
+      }
+    }
     stage('L??: Speech Checkpoints tests') {
       when {
         anyOf {

docs/source/conf.py

@@ -61,6 +61,9 @@
     'ipadic',
     'psutil',
     'regex',
+    'PIL',
+    'boto3',
+    'taming',
 ]
 
 _skipped_autodoc_mock_imports = ['wrapt', 'numpy']
@@ -125,6 +128,8 @@
     'tts/tts_all.bib',
     'text_processing/text_processing_all.bib',
     'core/adapters/adapter_bib.bib',
+    'multimodal/mm_all.bib',
+    'vision/vision_all.bib',
 ]
 
 intersphinx_mapping = {

docs/source/index.rst

@@ -47,7 +47,7 @@ NVIDIA NeMo User Guide
    nlp/api
    nlp/megatron_onnx_export
    nlp/models
-   
+
 
 .. toctree::
    :maxdepth: 1
@@ -71,6 +71,23 @@ NVIDIA NeMo User Guide
    text_processing/g2p/g2p
    common/intro
 
+.. toctree::
+   :maxdepth: 3
+   :caption: Multimodal (MM)
+   :name: Multimodal
+
+   multimodal/mllm/intro
+   multimodal/vlm/intro
+   multimodal/text2img/intro
+   multimodal/nerf/intro
+   multimodal/api
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Vision
+   :name: vision
+
+   vision/intro
 
 .. toctree::
    :maxdepth: 3