centre-for-humanities-computing · rlrs · Feb 8, 2024 · Feb 6, 2024 · Feb 6, 2024 · Feb 6, 2024
diff --git a/.gitmodules b/.gitmodules
@@ -1,4 +1,4 @@
 [submodule "llm-foundry"]
-	path = llm-foundry
+	path = LUMI/llm-foundry
 	url = https://github.com/rlrs/llm-foundry
 	branch = lumi
diff --git a/pyproject.toml → data-processing/pyproject.toml b/pyproject.toml → data-processing/pyproject.toml
diff --git a/...ata_processing/convert_dagw_to_jsonlgz.py → ...essing/scripts/convert_dagw_to_jsonlgz.py b/...ata_processing/convert_dagw_to_jsonlgz.py → ...essing/scripts/convert_dagw_to_jsonlgz.py
diff --git a/...a_processing/convert_danews_to_jsonlgz.py → ...sing/scripts/convert_danews_to_jsonlgz.py b/...a_processing/convert_danews_to_jsonlgz.py → ...sing/scripts/convert_danews_to_jsonlgz.py
diff --git a/scripts/data/jsonl_to_mds.py → data-processing/scripts/jsonl_to_mds.py b/scripts/data/jsonl_to_mds.py → data-processing/scripts/jsonl_to_mds.py
diff --git a/...essing/nat/convert_nat_to_dolma_format.py → ...cripts/nat/convert_nat_to_dolma_format.py b/...essing/nat/convert_nat_to_dolma_format.py → ...cripts/nat/convert_nat_to_dolma_format.py
diff --git a/...ssing/twitter/conversations_to_threads.py → ...ripts/twitter/conversations_to_threads.py b/...ssing/twitter/conversations_to_threads.py → ...ripts/twitter/conversations_to_threads.py
diff --git a/...itter/convert_twitter_to_conversations.py → ...itter/convert_twitter_to_conversations.py b/...itter/convert_twitter_to_conversations.py → ...itter/convert_twitter_to_conversations.py
diff --git a/src/dfm/common/__init__.py → data-processing/src/__init__.py b/src/dfm/common/__init__.py → data-processing/src/__init__.py
diff --git a/...on/data_cleaning/ccnet_text_normalizer.py → data-processing/src/ccnet_text_normalizer.py b/...on/data_cleaning/ccnet_text_normalizer.py → data-processing/src/ccnet_text_normalizer.py
diff --git a/src/dfm/common/data_cleaning/__init__.py → ...-processing/src/dolma_taggers/__init__.py b/src/dfm/common/data_cleaning/__init__.py → ...-processing/src/dolma_taggers/__init__.py
diff --git a/...cleaning/dolma_taggers/language_scandi.py → ...sing/src/dolma_taggers/language_scandi.py b/...cleaning/dolma_taggers/language_scandi.py → ...sing/src/dolma_taggers/language_scandi.py
diff --git a/...data_cleaning/dolma_taggers/perplexity.py → ...rocessing/src/dolma_taggers/perplexity.py b/...data_cleaning/dolma_taggers/perplexity.py → ...rocessing/src/dolma_taggers/perplexity.py
diff --git a/llm-foundry b/llm-foundry
diff --git a/src/dfm/__init__.py b/src/dfm/__init__.py
diff --git a/src/dfm/common/data_cleaning/dolma_taggers/__init__.py b/src/dfm/common/data_cleaning/dolma_taggers/__init__.py
diff --git a/src/dfm/common/data_cleaning/placeholder_common_file.py b/src/dfm/common/data_cleaning/placeholder_common_file.py
diff --git a/src/dfm/common/model_training/placeholder_common_file copy.py b/src/dfm/common/model_training/placeholder_common_file copy.py
diff --git a/src/dfm/projects/experiments/data_cleaning/placeholder.py b/src/dfm/projects/experiments/data_cleaning/placeholder.py
diff --git a/src/dfm/projects/experiments/model_optimisation/placeholder copy.py b/src/dfm/projects/experiments/model_optimisation/placeholder copy.py
diff --git a/src/dfm/test_sample.py b/src/dfm/test_sample.py
diff --git a/scripts/lumi/README.md → training/README.md b/scripts/lumi/README.md → training/README.md
@@ -1,12 +1,14 @@
 # Model training on LUMI
 
 ## Dataset preparation
+
 From a jsonl file (such as da-gigaword), something like `python scripts/data/convert_dataset_json.py --path /path/to/da-gigaword.jsonl.tar.gz  --out_root ./da-gigaword-mds --concat_tokens 4096 --tokenizer mistralai/Mistral-7B-v0.1 --test_size 0.02` will generate the necessary Mosaic streaming dataset. Takes ~2 hours for da-gigaword, which is a bit slow. When done, copy this folder to LUMI scratch and configure data path in the training YAML, e.g. `scripts/lumi/yamls/continue-mistral-7b.yaml`.
 
 ## LUMI setup and training
+
 1. SSH into LUMI
-3. Enter project: `cd /scratch/project_465000670/danish-foundation-models`
-2. Enter container: `singularity run --cleanenv --bind /scratch/project_465000670/ /project/project_465000670/pytorch_rocm5.7_ubuntu22.04_py3.10_pytorch_2.0.1.sif`
-5. Set up virtual environment: `./scripts/lumi/make_venv.sh`
-6. Exit container
-7. Run training: `./scripts/lumi/continue_mistral_mosaic.sh`
+2. Enter project: `cd /scratch/project_465000670/danish-foundation-models`
+3. Enter container: `singularity run --cleanenv --bind /scratch/project_465000670/ /project/project_465000670/pytorch_rocm5.7_ubuntu22.04_py3.10_pytorch_2.0.1.sif`
+4. Set up virtual environment: `./scripts/lumi/make_venv.sh`
+5. Exit container
+6. Run training: `./scripts/lumi/continue_mistral_mosaic.sh`
diff --git a/scripts/lumi/continue_mistral_mosaic.sh → training/continue_mistral_mosaic.sh b/scripts/lumi/continue_mistral_mosaic.sh → training/continue_mistral_mosaic.sh
diff --git a/scripts/lumi/make_venv.sh → training/make_venv.sh b/scripts/lumi/make_venv.sh → training/make_venv.sh
diff --git a/scripts/lumi/mosaic_in_container.sh → training/mosaic_in_container.sh b/scripts/lumi/mosaic_in_container.sh → training/mosaic_in_container.sh
diff --git a/scripts/lumi/requirements.txt → training/requirements.txt b/scripts/lumi/requirements.txt → training/requirements.txt
diff --git a/scripts/lumi/yamls/continue-mistral-7b.yaml → training/yamls/continue-mistral-7b.yaml b/scripts/lumi/yamls/continue-mistral-7b.yaml → training/yamls/continue-mistral-7b.yaml