From 9be6864892f927a190d91ec50a54fb15042c0c04 Mon Sep 17 00:00:00 2001 From: Hien To Date: Thu, 11 Jul 2024 10:44:51 +0700 Subject: [PATCH 1/8] Add Dockerfile for github runner --- .github/runners/Dockerfile | 100 +++++++++++++++++++++++++++++++++++++ .github/runners/start.sh | 21 ++++++++ 2 files changed, 121 insertions(+) create mode 100644 .github/runners/Dockerfile create mode 100644 .github/runners/start.sh diff --git a/.github/runners/Dockerfile b/.github/runners/Dockerfile new file mode 100644 index 0000000..546d68f --- /dev/null +++ b/.github/runners/Dockerfile @@ -0,0 +1,100 @@ +FROM nvidia/cuda:12.1.1-cudnn8-devel-ubuntu22.04 AS clang18_image + +# Install dependencies +RUN apt-get -qq update; \ + apt-get install -qqy --no-install-recommends \ + gnupg2 wget ca-certificates apt-transport-https \ + autoconf automake cmake dpkg-dev file make patch libc6-dev + +# Install LLVM +#RUN echo "deb https://apt.llvm.org/jammy llvm-toolchain-jammy-18 main" \ +# > /etc/apt/sources.list.d/llvm.list && \ +# wget -qO /etc/apt/trusted.gpg.d/llvm.asc \ +# https://apt.llvm.org/llvm-snapshot.gpg.key && \ +# apt-get update && \ +# apt-get install -y -t llvm-toolchain-jammy-18 clang-18 clangd-18 clang-tidy-18 clang-format-18 lld-18 libc++-18-dev libc++abi-18-dev && \ +# for f in /usr/lib/llvm-18/bin/*; do ln -sf "$f" /usr/bin; done && \ +# rm -rf /var/lib/apt/lists/* + +FROM clang18_image AS base_image + +# Add python PPA +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt update && apt install -y software-properties-common \ + && add-apt-repository ppa:deadsnakes/ppa -y + +# Install packages +RUN apt-get update && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + python3.11 \ + #python3-pip \ + python3.11-dev \ + zsh \ + nvtop \ + btop \ + tmux \ + git \ + curl \ + ca-certificates \ + gettext \ + unzip \ + fd-find \ + nodejs \ + npm \ + && rm -rf /var/lib/apt/lists/* + +# Set clang as the default compiler +#RUN ln -sf /usr/bin/clang /usr/bin/cc \ +# && ln -sf /usr/bin/clang++ /usr/bin/c++ \ +# && cc --version \ +# && c++ --version + +# Update the alternatives for Python 3.12 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 100 + +# Fix cuda clang issue +# Command to append content to the clangd config file +RUN mkdir -p /root/.config/clangd && \ + echo "CompileFlags:" >> /root/.config/clangd/config.yaml && \ + echo " Add:" >> /root/.config/clangd/config.yaml && \ + echo " - --cuda-gpu-arch=sm_89" >> /root/.config/clangd/config.yaml && \ + echo " Remove:" >> /root/.config/clangd/config.yaml && \ + echo " - --generate-code=arch=*" >> /root/.config/clangd/config.yaml && \ + echo " - -forward-unknown-to-host-compiler" >> /root/.config/clangd/config.yaml + +ENV HOME=/home/runner + +RUN mkdir -p /home/runner + +ARG RUNNER_VERSION=2.317.0 + +ARG RUNNER_UID=1000 +ARG DOCKER_GID=1001 + +RUN adduser --disabled-password --gecos "" --uid $RUNNER_UID runner \ + && groupadd docker --gid $DOCKER_GID \ + && usermod -aG sudo runner \ + && usermod -aG docker runner \ + && echo "%sudo ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers \ + && echo "Defaults env_keep += \"DEBIAN_FRONTEND\"" >> /etc/sudoers + +# cd into the user directory, download and unzip the github actions runner +RUN cd /home/runner && mkdir actions-runner && cd actions-runner \ + && curl -O -L https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz \ + && tar xzf ./actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz + +RUN chown -R runner:runner /home/runner && /home/runner/actions-runner/bin/installdependencies.sh + +ADD ./start.sh /home/runner/start.sh + +RUN chmod +x /home/runner/start.sh + +# Add /usr/local/cuda-11.7/compat to LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH=/usr/local/cuda-12.1/compat${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} + +ENTRYPOINT ["/bin/bash", "/home/runner/start.sh"] + +USER runner \ No newline at end of file diff --git a/.github/runners/start.sh b/.github/runners/start.sh new file mode 100644 index 0000000..84d3c3d --- /dev/null +++ b/.github/runners/start.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +RUNNER_REPO=$RUNNER_REPO +RUNNER_PAT=$RUNNER_PAT +RUNNER_GROUP=$RUNNER_GROUP +RUNNER_LABELS=$RUNNER_LABELS +RUNNER_NAME=$(hostname) + +cd /home/runner/actions-runner + +./config.sh --unattended --replace --url https://github.com/${RUNNER_REPO} --pat ${RUNNER_PAT} --name ${RUNNER_NAME} --runnergroup ${RUNNER_GROUP} --labels ${RUNNER_LABELS} --work /home/runner/actions-runner/_work + +cleanup() { + echo "Removing runner..." + ./config.sh remove --unattended --pat ${RUNNER_PAT} +} + +trap 'cleanup; exit 130' INT +trap 'cleanup; exit 143' TERM + +./run.sh & wait $! \ No newline at end of file From a12e4b980b1667aeac08e96086b5e0adfc0970f1 Mon Sep 17 00:00:00 2001 From: Hien To Date: Fri, 12 Jul 2024 12:05:47 +0700 Subject: [PATCH 2/8] Add CICD --- .github/workflows/test-models.yml | 42 +++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/test-models.yml diff --git a/.github/workflows/test-models.yml b/.github/workflows/test-models.yml new file mode 100644 index 0000000..2f99495 --- /dev/null +++ b/.github/workflows/test-models.yml @@ -0,0 +1,42 @@ +name: Test - Models +on: + workflow_dispatch: + inputs: + model_id: + description: 'Model ID on huggingface, for example: jan-hq/Jan-Llama3-0708' + required: true + default: jan-hq/Jan-Llama3-0708 + type: string + dataset_id: + description: 'Dataset ID on huggingface, for example: jan-hq/instruction-speech-conversation-test' + required: true + default: jan-hq/instruction-speech-conversation-test + type: string + extra_args: + description: 'Extra arguments for python command, for example:--mode audio --num_rows 5' + required: false + default: "--mode audio --num_rows 5" + type: string + +jobs: + run-test: + runs-on: research + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.11' + + - name: Install dependencies + working-directory: ./tests + run: | + python3 -m pip install --upgrade pip + pip3 install -r requirements.txt + + - name: Run tests + working-directory: ./tests + run: | + python3 test_case.py --model_dir ${{ github.event.inputs.model_id }} --data_dir ${{ github.event.inputs.dataset_id }} ${{ github.event.inputs.extra_args }} \ No newline at end of file From a6eeb113048069e525eef9306320047d81891170 Mon Sep 17 00:00:00 2001 From: bachvudinh Date: Fri, 12 Jul 2024 16:27:03 +0700 Subject: [PATCH 3/8] debug --- tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index eb01ff8..6f4ee6f 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,5 +1,5 @@ datasets==2.20.0 -torch=2.2.0 +torch==2.2.0 transformers vllm huggingface_hub==0.23.4 From 752cff6ffdd6493c8cbe7a77dcf2bea936a5c3d1 Mon Sep 17 00:00:00 2001 From: bachvudinh Date: Fri, 12 Jul 2024 16:42:50 +0700 Subject: [PATCH 4/8] debug --- tests/inference_vllm.py | 47 ----------------------------------------- tests/requirements.txt | 2 +- tests/test_case.py | 6 +++--- 3 files changed, 4 insertions(+), 51 deletions(-) delete mode 100644 tests/inference_vllm.py diff --git a/tests/inference_vllm.py b/tests/inference_vllm.py deleted file mode 100644 index f080a5b..0000000 --- a/tests/inference_vllm.py +++ /dev/null @@ -1,47 +0,0 @@ -from accelerate.utils import merge_fsdp_weights -from huggingface_hub import snapshot_download -from vllm import LLM, SamplingParams -from transformers import AutoTokenizer -from datasets import load_dataset -import pandas as pd -# Global variable -save_dir_output = 'output/Jan-Llama3-0708-Result.csv' -sampling_params = SamplingParams(temperature=0.0, max_tokens=200) -local_dir = "jan-hq/Jan-Llama3-0708" -snapshot_download("jan-hq/Jan-Llama3-0708", local_dir=local_dir, max_workers=64) -# Model loading using vllm -tokenizer = AutoTokenizer.from_pretrained("jan-hq/llama-3-sound-init") -llm = LLM(local_dir, tokenizer="jan-hq/llama-3-sound-init") -dataset = load_dataset("jan-hq/instruction-speech-conversation-test", cache_dir="/.cache/")['train'] -def vllm_inference(sample_id): - sound_messages = dataset[sample_id]['sound_convo'][0] - expected_output_messages = dataset[sample_id]['sound_convo'][1] - - sound_input_str = tokenizer.apply_chat_template([sound_messages], tokenize=False, add_generation_prompt=True) - text_input_str = dataset[sample_id]['prompt'] - output_based_on_text = tokenizer.apply_chat_template([expected_output_messages], tokenize=False) - - outputs = llm.generate(sound_input_str, sampling_params) - output_based_on_text = outputs[0].outputs[0].text - output_token_ids = outputs[0].outputs[0].token_ids - - print("-"*50) - print("Text input: ", text_input_str) - print("-"*50) - print("Text output: ", output_str) - print("-"*50) - print("Expected output: ", expected_output_str) - print("-"*50) - print("Output token ids: ", output_token_ids) - print("-"*50) - - return text_input_str, output_based_on_sound, output_based_on_text, output_token_ids -ouput_df = pd.DataFrame() -for i in range(len(dataset)): - text_input_str, output_based_on_sound, output_based_on_text, output_token_ids = vllm_inference(i) - # add to dictionary - output_df['text_input'].append(text_input_str) - output_df['output_based_on_sound'].append(output_based_on_sound) - output_df['output_based_on_text'].append(output_based_on_text) - output_df['output_token_ids'].append(output_token_ids) -output_df.to_csv(save_dir_output, index=False, encoding='utf-8') \ No newline at end of file diff --git a/tests/requirements.txt b/tests/requirements.txt index 6f4ee6f..e41cf80 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,5 +1,5 @@ datasets==2.20.0 -torch==2.2.0 +torch==2.3.0 transformers vllm huggingface_hub==0.23.4 diff --git a/tests/test_case.py b/tests/test_case.py index d77ee2d..f98fd97 100644 --- a/tests/test_case.py +++ b/tests/test_case.py @@ -12,8 +12,8 @@ def parse_arguments(): parser = argparse.ArgumentParser(description="Run inference on a Sound-To-Text Model.") - parser.add_argument("--model_dir", type=str, required=True, help="Hugging Face model link or local_dir") - parser.add_argument("--model_save_dir", type=str, required=True, help="Local directory that model is saved") + parser.add_argument("--model_dir", type=str, default="jan-hq/Jan-Llama3-0708", help="Hugging Face model link or local_dir") + parser.add_argument("--model_save_dir", type=str, required=False, help="Local directory that model is saved") parser.add_argument("--data_dir", type=str, required=True, help="Hugging Face model repository link or Data path") parser.add_argument("--mode", type=str, default="audio", help="Mode of the model (audio or text)") parser.add_argument("--num_rows", type=int, default=5, help="Number of dataset rows to process") @@ -31,7 +31,7 @@ def setUpClass(cls): os.makedirs(args.output_file) cls.sampling_params = SamplingParams(temperature=0.0, max_tokens=1024, skip_special_tokens=False) model_dir = "" - if os.path.exists(args.model_save_dir): + if args.model_save_dir: model_dir = args.model_save_dir else: # Download model From 1e56ce1cdc6fb56dc07adcd22b68424947c485f4 Mon Sep 17 00:00:00 2001 From: bachvudinh Date: Fri, 12 Jul 2024 18:04:27 +0700 Subject: [PATCH 5/8] delete model_save_dir --- tests/test_case.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/tests/test_case.py b/tests/test_case.py index f98fd97..0095aaf 100644 --- a/tests/test_case.py +++ b/tests/test_case.py @@ -13,7 +13,6 @@ def parse_arguments(): parser = argparse.ArgumentParser(description="Run inference on a Sound-To-Text Model.") parser.add_argument("--model_dir", type=str, default="jan-hq/Jan-Llama3-0708", help="Hugging Face model link or local_dir") - parser.add_argument("--model_save_dir", type=str, required=False, help="Local directory that model is saved") parser.add_argument("--data_dir", type=str, required=True, help="Hugging Face model repository link or Data path") parser.add_argument("--mode", type=str, default="audio", help="Mode of the model (audio or text)") parser.add_argument("--num_rows", type=int, default=5, help="Number of dataset rows to process") @@ -30,19 +29,14 @@ def setUpClass(cls): if not os.path.exists(args.output_file): os.makedirs(args.output_file) cls.sampling_params = SamplingParams(temperature=0.0, max_tokens=1024, skip_special_tokens=False) - model_dir = "" - if args.model_save_dir: - model_dir = args.model_save_dir + # Download model + if not os.path.exists(args.model_dir): + snapshot_download(args.model_dir, local_dir=args.model_dir, max_workers=64) else: - # Download model - if not os.path.exists(args.model_dir): - snapshot_download(args.model_dir, local_dir=args.model_dir, max_workers=64) - else: - print(f"Found {args.model_dir}. Skipping download.") - model_dir = args.model_dir + print(f"Found {args.model_dir}. Skipping download.") # Model loading using vllm - cls.tokenizer = AutoTokenizer.from_pretrained(model_dir) - cls.llm = LLM(model_dir, tokenizer=model_dir) + cls.tokenizer = AutoTokenizer.from_pretrained(args.model_dir) + cls.llm = LLM(args.model_dir, tokenizer=args.model_dir) # Load dataset cls.dataset = load_dataset(args.data_dir, cache_dir=".cache/")['train'] From 8ada9773db3aea717f3b60f5025982b78384879c Mon Sep 17 00:00:00 2001 From: bachvudinh Date: Fri, 12 Jul 2024 18:17:19 +0700 Subject: [PATCH 6/8] update ab path to save model and dataset --- tests/test_case.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_case.py b/tests/test_case.py index 0095aaf..da95f5b 100644 --- a/tests/test_case.py +++ b/tests/test_case.py @@ -14,6 +14,7 @@ def parse_arguments(): parser = argparse.ArgumentParser(description="Run inference on a Sound-To-Text Model.") parser.add_argument("--model_dir", type=str, default="jan-hq/Jan-Llama3-0708", help="Hugging Face model link or local_dir") parser.add_argument("--data_dir", type=str, required=True, help="Hugging Face model repository link or Data path") + parser.add_argument("--cache_dir", type=str, default=".", help="Absolute path to save the model and dataset") parser.add_argument("--mode", type=str, default="audio", help="Mode of the model (audio or text)") parser.add_argument("--num_rows", type=int, default=5, help="Number of dataset rows to process") parser.add_argument("--output_file", type=str, default="output/", help="Output file path") @@ -31,15 +32,15 @@ def setUpClass(cls): cls.sampling_params = SamplingParams(temperature=0.0, max_tokens=1024, skip_special_tokens=False) # Download model if not os.path.exists(args.model_dir): - snapshot_download(args.model_dir, local_dir=args.model_dir, max_workers=64) + snapshot_download(args.model_dir, local_dir=args.cache_dir, max_workers=64) else: print(f"Found {args.model_dir}. Skipping download.") # Model loading using vllm - cls.tokenizer = AutoTokenizer.from_pretrained(args.model_dir) - cls.llm = LLM(args.model_dir, tokenizer=args.model_dir) + cls.tokenizer = AutoTokenizer.from_pretrained(args.cache_dir) + cls.llm = LLM(args.cache_dir, tokenizer=args.cache_dir) # Load dataset - cls.dataset = load_dataset(args.data_dir, cache_dir=".cache/")['train'] + cls.dataset = load_dataset(args.data_dir, cache_dir=args.cache_dir)['train'] cls.num_rows = min(args.num_rows, len(cls.dataset)) cls.inference_results = [] if args.mode == "audio": From d74e8c7b8535e0ca9d18b25d27805630a3ce3c3c Mon Sep 17 00:00:00 2001 From: Hien To Date: Fri, 12 Jul 2024 21:45:12 +0700 Subject: [PATCH 7/8] Change CI runner Dockerfile --- .github/runners/Dockerfile | 70 +++++-------------------------- .github/workflows/test-models.yml | 5 --- 2 files changed, 11 insertions(+), 64 deletions(-) diff --git a/.github/runners/Dockerfile b/.github/runners/Dockerfile index 546d68f..6f5827d 100644 --- a/.github/runners/Dockerfile +++ b/.github/runners/Dockerfile @@ -1,69 +1,21 @@ -FROM nvidia/cuda:12.1.1-cudnn8-devel-ubuntu22.04 AS clang18_image +FROM docker.io/pytorch/pytorch:2.3.0-cuda12.1-cudnn8-runtime -# Install dependencies -RUN apt-get -qq update; \ - apt-get install -qqy --no-install-recommends \ - gnupg2 wget ca-certificates apt-transport-https \ - autoconf automake cmake dpkg-dev file make patch libc6-dev - -# Install LLVM -#RUN echo "deb https://apt.llvm.org/jammy llvm-toolchain-jammy-18 main" \ -# > /etc/apt/sources.list.d/llvm.list && \ -# wget -qO /etc/apt/trusted.gpg.d/llvm.asc \ -# https://apt.llvm.org/llvm-snapshot.gpg.key && \ -# apt-get update && \ -# apt-get install -y -t llvm-toolchain-jammy-18 clang-18 clangd-18 clang-tidy-18 clang-format-18 lld-18 libc++-18-dev libc++abi-18-dev && \ -# for f in /usr/lib/llvm-18/bin/*; do ln -sf "$f" /usr/bin; done && \ -# rm -rf /var/lib/apt/lists/* - -FROM clang18_image AS base_image - -# Add python PPA -ARG DEBIAN_FRONTEND=noninteractive - -RUN apt update && apt install -y software-properties-common \ - && add-apt-repository ppa:deadsnakes/ppa -y - -# Install packages -RUN apt-get update && apt-get upgrade -y \ +RUN apt-get update \ && apt-get install -y --no-install-recommends \ build-essential \ cmake \ - python3.11 \ - #python3-pip \ - python3.11-dev \ - zsh \ - nvtop \ - btop \ - tmux \ - git \ - curl \ - ca-certificates \ - gettext \ + sudo \ unzip \ - fd-find \ - nodejs \ - npm \ + curl \ + wget \ + git \ + git-lfs \ + jq \ && rm -rf /var/lib/apt/lists/* -# Set clang as the default compiler -#RUN ln -sf /usr/bin/clang /usr/bin/cc \ -# && ln -sf /usr/bin/clang++ /usr/bin/c++ \ -# && cc --version \ -# && c++ --version - -# Update the alternatives for Python 3.12 -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 100 - -# Fix cuda clang issue -# Command to append content to the clangd config file -RUN mkdir -p /root/.config/clangd && \ - echo "CompileFlags:" >> /root/.config/clangd/config.yaml && \ - echo " Add:" >> /root/.config/clangd/config.yaml && \ - echo " - --cuda-gpu-arch=sm_89" >> /root/.config/clangd/config.yaml && \ - echo " Remove:" >> /root/.config/clangd/config.yaml && \ - echo " - --generate-code=arch=*" >> /root/.config/clangd/config.yaml && \ - echo " - -forward-unknown-to-host-compiler" >> /root/.config/clangd/config.yaml +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ + unzip awscliv2.zip && \ + ./aws/install ENV HOME=/home/runner diff --git a/.github/workflows/test-models.yml b/.github/workflows/test-models.yml index 2f99495..3e85833 100644 --- a/.github/workflows/test-models.yml +++ b/.github/workflows/test-models.yml @@ -25,11 +25,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.11' - - name: Install dependencies working-directory: ./tests run: | From 02af579d30e2acf76317e77fc4116f2118a5f529 Mon Sep 17 00:00:00 2001 From: bachvudinh Date: Sat, 13 Jul 2024 09:30:20 +0000 Subject: [PATCH 8/8] add CI-CD test for dev branch --- .github/workflows/test-branch.yml | 41 +++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/test-branch.yml diff --git a/.github/workflows/test-branch.yml b/.github/workflows/test-branch.yml new file mode 100644 index 0000000..a02e536 --- /dev/null +++ b/.github/workflows/test-branch.yml @@ -0,0 +1,41 @@ +name: Test - Models + +on: + push: + branches: + - 'CI-CD/bach' + workflow_dispatch: + inputs: + model_id: + description: 'Model ID on huggingface, for example: jan-hq/Jan-Llama3-0708' + required: true + default: jan-hq/Jan-Llama3-0708 + type: string + dataset_id: + description: 'Dataset ID on huggingface, for example: jan-hq/instruction-speech-conversation-test' + required: true + default: jan-hq/instruction-speech-conversation-test + type: string + extra_args: + description: 'Extra arguments for python command, for example:--mode audio --num_rows 5' + required: false + default: "--mode audio --num_rows 5" + type: string + +jobs: + run-test: + runs-on: research + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install dependencies + working-directory: ./tests + run: | + python3 -m pip install --upgrade pip + pip3 install -r requirements.txt + + - name: Run tests + working-directory: ./tests + run: | + python3 test_case.py --model_dir ${{ github.event.inputs.model_id || 'jan-hq/Jan-Llama3-0708' }} --data_dir ${{ github.event.inputs.dataset_id || 'jan-hq/instruction-speech-conversation-test' }} ${{ github.event.inputs.extra_args || '--mode audio --num_rows 5' }} \ No newline at end of file