causal_lm_cpp.yml

name: causal_lm_cpp
on:
  pull_request:
    paths:
      - .github/workflows/causal_lm_cpp.yml
      - src/**
      - samples/**
      - thirdparty/openvino_tokenizers
      - "!**.md"
permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  l_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2024.3.0-15805-6138d624dc1/l_openvino_toolkit_ubuntu20_2024.3.0.dev20240626_x86_64.tgz
  w_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2024.3.0-15805-6138d624dc1/w_openvino_toolkit_windows_2024.3.0.dev20240626_x86_64.zip
jobs:
  cpp-multinomial-greedy_causal_lm-ubuntu:
    runs-on: ubuntu-20.04-8-cores
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Install OpenVINO
        run: |
          mkdir ./ov/
          curl ${{ env.l_ov_link }} | tar --directory ./ov/ --strip-components 1 -xz
          sudo ./ov/install_dependencies/install_openvino_dependencies.sh
      - name: Download, convert and build
        run: |
          source ./ov/setupvars.sh
          python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          optimum-cli export openvino --trust-remote-code --weight-format fp16 --model openlm-research/open_llama_3b_v2 open_llama_3b_v2
          cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
          cmake --build ./build/ --config Release -j
      - run: >
          . ./ov/setupvars.sh
          && PYTHONPATH=./build/:$PYTHONPATH timeout 25s
          ./build/samples/cpp/multinomial_causal_lm/multinomial_causal_lm ./open_llama_3b_v2/ a
      - run: >
          . ./ov/setupvars.sh
          && PYTHONPATH=./build/:$PYTHONPATH timeout 25s
          ./samples/python/multinomial_causal_lm/multinomial_causal_lm.py ./open_llama_3b_v2/ b
      - run: >
          . ./ov/setupvars.sh
          && export PYTHONPATH=./build/:$PYTHONPATH
          && timeout 25s ./build/samples/cpp/greedy_causal_lm/greedy_causal_lm ./open_llama_3b_v2/ "return 0"
          | diff <(timeout 25s samples/python/greedy_causal_lm/greedy_causal_lm.py ./open_llama_3b_v2/ "return 0") -

  cpp-beam_search_causal_lm-ubuntu:
    strategy:
      matrix:
        executable:
          [
            ./build/samples/cpp/beam_search_causal_lm/beam_search_causal_lm,
            python ./samples/python/beam_search_causal_lm/beam_search_causal_lm.py,
          ]
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Install OpenVINO
        run: |
          mkdir ./ov/
          curl ${{ env.l_ov_link }} | tar --directory ./ov/ --strip-components 1 -xz
          sudo ./ov/install_dependencies/install_openvino_dependencies.sh
      - name: Download, convert and build
        run: |
          source ./ov/setupvars.sh
          python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
          cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
          cmake --build ./build/ --config Release -j
      - name: Compare
        run: |
          source ./ov/setupvars.sh
          export PYTHONPATH=./build/:$PYTHONPATH  # C++ ignores that

          timeout 25s ${{ matrix.executable }} ./TinyLlama-1.1B-Chat-v1.0/ "Why is the Sun yellow?" > ./pred.txt
          python -c "
          import transformers
          with open('pred.txt', 'r') as file:
              predictions = file.read()
          tokenizer = transformers.LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0')
          tokenized = tokenizer('Why is the Sun yellow?', return_tensors='pt')
          for beam in transformers.LlamaForCausalLM.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0').generate(**tokenized, num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9, do_sample=False):
              ref = ': ' + tokenizer.decode(beam[tokenized['input_ids'].numel():], skip_special_tokens=True)
              idx = predictions.find(ref)
              if -1 == idx:
                  raise RuntimeError(f'Missing "{ref=}" from predictions')
              predictions = predictions[:idx] + predictions[idx + len(ref):]
          "
          echo "Why is the Sun yellow?" passed

          timeout 25s ${{ matrix.executable }} ./TinyLlama-1.1B-Chat-v1.0/ 69 > ./pred.txt
          python -c "
          import transformers
          with open('pred.txt', 'r') as file:
              predictions = file.read()
          tokenizer = transformers.LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0')
          tokenized = tokenizer('69', return_tensors='pt')
          for beam in transformers.LlamaForCausalLM.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0').generate(**tokenized, num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9, do_sample=False):
              ref = ': ' + tokenizer.decode(beam[tokenized['input_ids'].numel():], skip_special_tokens=True)
              idx = predictions.find(ref)
              if -1 == idx:
                  raise RuntimeError(f'Missing "{ref=}" from predictions')
              predictions = predictions[:idx] + predictions[idx + len(ref):]
          "
          echo 69 passed

          timeout 25s ${{ matrix.executable }} ./TinyLlama-1.1B-Chat-v1.0/ Hi > ./pred.txt
          python -c "
          import transformers
          with open('pred.txt', 'r') as file:
              predictions = file.read()
          tokenizer = transformers.LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0')
          tokenized = tokenizer('Hi', return_tensors='pt')
          for beam in transformers.LlamaForCausalLM.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0').generate(**tokenized, num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9, do_sample=False):
              ref = ': ' + tokenizer.decode(beam[tokenized['input_ids'].numel():], skip_special_tokens=True)
              idx = predictions.find(ref)
              if -1 == idx:
                  raise RuntimeError(f'Missing "{ref=}" from predictions')
              predictions = predictions[:idx] + predictions[idx + len(ref):]
          "
          echo "Hi" passed

          timeout 25s ${{ matrix.executable }} ./TinyLlama-1.1B-Chat-v1.0/ "return 0" > ./pred.txt
          python -c "
          import transformers
          with open('pred.txt', 'r') as file:
              predictions = file.read()
          tokenizer = transformers.LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0')
          tokenized = tokenizer('return 0', return_tensors='pt')
          for beam in transformers.LlamaForCausalLM.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0').generate(**tokenized, num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9, do_sample=False):
              ref = ': ' + tokenizer.decode(beam[tokenized['input_ids'].numel():], skip_special_tokens=True)
              idx = predictions.find(ref)
              if -1 == idx:
                  raise RuntimeError(f'Missing "{ref=}" from predictions')
              predictions = predictions[:idx] + predictions[idx + len(ref):]
          "
          echo "return 0" passed

          timeout 25s ${{ matrix.executable }} ./TinyLlama-1.1B-Chat-v1.0/ "你好！ 你好嗎？" > ./pred.txt
          python -c "
          import transformers
          with open('pred.txt', 'r') as file:
              predictions = file.read()
          tokenizer = transformers.LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0')
          tokenized = tokenizer('你好！ 你好嗎？', return_tensors='pt')
          for beam in transformers.LlamaForCausalLM.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0').generate(**tokenized, num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9, do_sample=False):
              ref = ': ' + tokenizer.decode(beam[tokenized['input_ids'].numel():], skip_special_tokens=True)
              idx = predictions.find(ref)
              if -1 == idx:
                  raise RuntimeError(f'Missing "{ref=}" from predictions')
              predictions = predictions[:idx] + predictions[idx + len(ref):]
          "
          echo "你好！ 你好嗎？" passed

          timeout 1m ${{ matrix.executable }} ./TinyLlama-1.1B-Chat-v1.0/ "Alan Turing was a" "return 0" "你好！ 你好嗎？" > ./pred.txt
          python -c "
          import transformers
          with open('pred.txt', 'r') as file:
              predictions = file.read()
          tokenizer = transformers.LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0')
          prompts = [
            'Alan Turing was a',
            'return 0',
            '你好！ 你好嗎？'
          ]
          for prompt in prompts:
            tokenized = tokenizer(prompt, return_tensors='pt')
            for beam in transformers.LlamaForCausalLM.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0').generate(**tokenized, num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9, do_sample=False):
                ref = ': ' + tokenizer.decode(beam[tokenized['input_ids'].numel():], skip_special_tokens=True)
                idx = predictions.find(ref)
                if -1 == idx:
                    raise RuntimeError(f'Missing "{ref=}" from predictions')
                predictions = predictions[:idx] + predictions[idx + len(ref):]
          "
          echo "Multi prompt" passed

  cpp-greedy_causal_lm-windows:
    runs-on: windows-latest
    defaults:
      run:
        shell: cmd
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - run: curl --output ov.zip ${{ env.w_ov_link }}
      - run: unzip -d ov ov.zip
      - run: dirs=(ov/*) && mv ov/*/* ov && rmdir "${dirs[@]}"
        shell: bash
      - name: Download, convert and build
        run: |
          call .\ov\setupvars.bat
          python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
          cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
          cmake --build ./build/ --config Release -j
      - run: >
          set PATH=.\build\openvino_genai\;%PATH%
          && call .\ov\setupvars.bat
          && .\build\samples\cpp\greedy_causal_lm\Release\greedy_causal_lm.exe .\TinyLlama-1.1B-Chat-v1.0\ 69 > .\cpp.txt
      - run: |
          echo import transformers > ref.py
          echo predictions = open('cpp.txt', 'r').read() >> ref.py
          echo tokenizer = transformers.AutoTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0', trust_remote_code=True) >> ref.py
          echo tokenized = tokenizer('69', return_tensors='pt') >> ref.py
          echo for beam in transformers.AutoModelForCausalLM.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0', trust_remote_code=True).generate(**tokenized, max_new_tokens=100, do_sample=False): >> ref.py
          echo     ref = tokenizer.decode(beam[tokenized['input_ids'].numel():], skip_special_tokens=True) >> ref.py
          echo     idx = predictions.find(ref) >> ref.py
          echo     if -1 == idx: >> ref.py
          echo         raise RuntimeError(f'Missing "{ref=}" from predictions') >> ref.py
          echo     predictions = predictions[:idx] + predictions[idx + len(ref):] >> ref.py
      - run: python ref.py
      - run: >
          set PATH=.\build\openvino_genai\;%PATH%
          && set "PYTHONPATH=./build/"
          && call .\ov\setupvars.bat
          && python samples\python\greedy_causal_lm\greedy_causal_lm.py .\TinyLlama-1.1B-Chat-v1.0\ 69 > .\py.txt
      - run: fc .\cpp.txt .\py.txt

  cpp-beam_search_causal_lm-Qwen-7B-Chat:
    runs-on: ubuntu-20.04-16-cores
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Install OpenVINO
        run: |
          mkdir ./ov/
          curl ${{ env.l_ov_link }} | tar --directory ./ov/ --strip-components 1 -xz
          sudo ./ov/install_dependencies/install_openvino_dependencies.sh
      - name: Download, convert and build
        run: |
          source ./ov/setupvars.sh
          python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          optimum-cli export openvino --trust-remote-code --weight-format fp16 --model Qwen/Qwen-7B-Chat Qwen-7B-Chat
          cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
          cmake --build ./build/ --config Release -j
      - run: >
          . ./ov/setupvars.sh
          && export PYTHONPATH=./build/:$PYTHONPATH
          && timeout 2m ./build/samples/cpp/greedy_causal_lm/greedy_causal_lm ./Qwen-7B-Chat/ 69 | diff <(timeout 2m samples/python/greedy_causal_lm/greedy_causal_lm.py ./Qwen-7B-Chat/ 69) -

  cpp-beam_search_causal_lm-Qwen1_5-7B-Chat:
    runs-on: ubuntu-20.04-16-cores
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Install OpenVINO
        run: |
          mkdir ./ov/
          curl ${{ env.l_ov_link }} | tar --directory ./ov/ --strip-components 1 -xz
          sudo ./ov/install_dependencies/install_openvino_dependencies.sh
      - name: Download, convert and build
        run: |
          source ./ov/setupvars.sh
          python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          optimum-cli export openvino --trust-remote-code --weight-format fp16 --model Qwen/Qwen1.5-7B-Chat Qwen1.5-7B-Chat
          cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
          cmake --build ./build/ --config Release -j
      - run: >
          . ./ov/setupvars.sh
          && export PYTHONPATH=./build/:$PYTHONPATH
          && timeout 50s ./build/samples/cpp/beam_search_causal_lm/beam_search_causal_lm ./Qwen1.5-7B-Chat/ "你好！"
          | diff <(timeout 50s ./samples/python/beam_search_causal_lm/beam_search_causal_lm.py ./Qwen1.5-7B-Chat/ "你好！") -

  cpp-beam_search_causal_lm-Phi-2:
    runs-on: ubuntu-20.04-16-cores
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Install OpenVINO
        run: |
          mkdir ./ov/
          curl ${{ env.l_ov_link }} | tar --directory ./ov/ --strip-components 1 -xz
          sudo ./ov/install_dependencies/install_openvino_dependencies.sh
      - name: Download, convert and build
        run: |
          source ./ov/setupvars.sh
          python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          optimum-cli export openvino --trust-remote-code --weight-format fp16 --model microsoft/phi-2 phi-2
          cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
          cmake --build ./build/ --config Release -j 15
      - run: >
          . ./ov/setupvars.sh
          && export PYTHONPATH=./build/:$PYTHONPATH
          && timeout 50s ./build/samples/cpp/beam_search_causal_lm/beam_search_causal_lm ./phi-2/ 69
          | diff <(timeout 50s ./samples/python/beam_search_causal_lm/beam_search_causal_lm.py ./phi-2/ 69) -

  cpp-beam_search_causal_lm-notus-7b-v1:
    runs-on: ubuntu-20.04-16-cores
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Install OpenVINO
        run: |
          mkdir ./ov/
          curl ${{ env.l_ov_link }} | tar --directory ./ov/ --strip-components 1 -xz
          sudo ./ov/install_dependencies/install_openvino_dependencies.sh
      - name: Download, convert and build
        run: |
          source ./ov/setupvars.sh
          python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          optimum-cli export openvino --trust-remote-code --weight-format fp16 --model argilla/notus-7b-v1 notus-7b-v1
          cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
          cmake --build ./build/ --config Release -j
      - run: >
          . ./ov/setupvars.sh
          && export PYTHONPATH=./build/:$PYTHONPATH
          && timeout 50s ./build/samples/cpp/beam_search_causal_lm/beam_search_causal_lm ./notus-7b-v1/ 69
          | diff <(timeout 50s ./samples/python/beam_search_causal_lm/beam_search_causal_lm.py ./notus-7b-v1/ 69) -

  cpp-speculative_decoding_lm-ubuntu:
    runs-on: ubuntu-20.04-16-cores
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Install OpenVINO
        run: |
          mkdir ./ov/
          curl ${{ env.l_ov_link }} | tar --directory ./ov/ --strip-components 1 -xz
          sudo ./ov/install_dependencies/install_openvino_dependencies.sh
      - name: Download, convert and build
        run: |
          source ./ov/setupvars.sh
          python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          optimum-cli export openvino --trust-remote-code --weight-format fp16 --model databricks/dolly-v2-3b dolly-v2-3b
          optimum-cli export openvino --trust-remote-code --weight-format fp16 --model databricks/dolly-v2-7b dolly-v2-7b
          cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
          cmake --build ./build/ --config Release -j
      - name: run and compare
        run: |
          source ./ov/setupvars.sh
          ./build/samples/cpp/speculative_decoding_lm/speculative_decoding_lm ./dolly-v2-3b/ ./dolly-v2-7b/ "Alan Turing was a" > predictions_speculative.txt
          ./build/samples/cpp/speculative_decoding_lm/speculative_decoding_lm ./dolly-v2-3b/ ./dolly-v2-7b/ "Alan Turing was a" > predictions_greedy.txt
          python -c "
          with open('predictions_greedy.txt', 'r') as f:
              predicted_greedy = f.readline()
          with open('predictions_speculative.txt', 'r') as f:
              predicted_speculative = f.readline()
          assert predicted_greedy == predicted_speculative
          "
          echo "Alan Turing was a" passed

  cpp-prompt_lookup_decoding_lm-ubuntu:
    runs-on: ubuntu-20.04-16-cores
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Install OpenVINO
        run: |
          mkdir ./ov/
          curl ${{ env.l_ov_link }} | tar --directory ./ov/ --strip-components 1 -xz
          sudo ./ov/install_dependencies/install_openvino_dependencies.sh
      - name: Download, convert and build
        run: |
          source ./ov/setupvars.sh
          python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
          cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
          cmake --build ./build/ --config Release -j
      - name: run and compare
        run: |
          source ./ov/setupvars.sh

          echo 'Code:```python
          def add(a, b):
              return a + b
          ```
          Question: Can you please add 2 and 3
          A:' > ./prompt.txt

          ./build/samples/cpp/prompt_lookup_decoding_lm/prompt_lookup_decoding_lm ./TinyLlama-1.1B-Chat-v1.0/ "$(<prompt.txt)" > predictions_prompt_lookup.txt
          ./build/samples/cpp/prompt_lookup_decoding_lm/prompt_lookup_decoding_lm ./TinyLlama-1.1B-Chat-v1.0/ "$(<prompt.txt)" > predictions_greedy.txt
          python -c "
          with open('predictions_greedy.txt', 'r') as f:
              predicted_greedy = f.readline()
          with open('predictions_prompt_lookup.txt', 'r') as f:
              predicted_prompt_lookup = f.readline()
          assert predicted_greedy == predicted_prompt_lookup
          "
          echo "Prompt lookup" passed

  cpp-Phi-1_5:
    runs-on: ubuntu-20.04-16-cores
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Install OpenVINO
        run: |
          mkdir ./ov/
          curl ${{ env.l_ov_link }} | tar --directory ./ov/ --strip-components 1 -xz
          sudo ./ov/install_dependencies/install_openvino_dependencies.sh
      - name: Download, convert and build
        run: |
          source ./ov/setupvars.sh
          python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          optimum-cli export openvino --trust-remote-code --weight-format fp16 --model microsoft/phi-1_5 phi-1_5
          cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
          cmake --build ./build/ --config Release -j 15
      - name: Run Generation
        run: |
          source ./ov/setupvars.sh
          timeout 50s ./build/samples/cpp/greedy_causal_lm/greedy_causal_lm ./phi-1_5/ "Alan Turing was a" > ./pred_greedy.txt
      - name: Compare
        run: |
          python -c "
          import transformers
          with open('pred_greedy.txt', 'r') as file:
              predictions = file.read()
          tokenizer = transformers.AutoTokenizer.from_pretrained('microsoft/phi-1_5')
          tokenized = tokenizer('Alan Turing was a', return_tensors='pt')
          for output in transformers.AutoModelForCausalLM.from_pretrained('microsoft/phi-1_5').generate(**tokenized, max_length=100, do_sample=False):
              ref = tokenizer.decode(output[tokenized['input_ids'].numel():], skip_special_tokens=True)
              idx = predictions.find(ref)
              if -1 == idx:
                  raise RuntimeError(f'Missing "{ref=}" from predictions')
              predictions = predictions[:idx] + predictions[idx + len(ref):]
          "
          echo Phi-1_5 passed
      - run: >
          . ./ov/setupvars.sh
          && export PYTHONPATH=./build/:$PYTHONPATH
          && timeout 50s samples/python/greedy_causal_lm/greedy_causal_lm.py ./phi-1_5/ "Alan Turing was a"
          | diff ./pred_greedy.txt -

  cpp-greedy_causal_lm-redpajama-3b-chat:
    runs-on: ubuntu-20.04-4-cores
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Install OpenVINO
        run: |
          mkdir ./ov/
          curl ${{ env.l_ov_link }} | tar --directory ./ov/ --strip-components 1 -xz
          sudo ./ov/install_dependencies/install_openvino_dependencies.sh
      - name: Download, convert and build
        run: |
          source ./ov/setupvars.sh
          python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
          optimum-cli export openvino --trust-remote-code --weight-format fp16 --model ikala/redpajama-3b-chat redpajama-3b-chat
          cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
          cmake --build ./build/ --config Release -j
      - run: source ./ov/setupvars.sh && convert_tokenizer ./redpajama-3b-chat/ --output ./redpajama-3b-chat/ --with-detokenizer --trust-remote-code
      - name: Run Generation
        run: |
          source ./ov/setupvars.sh

          timeout 50s ./build/samples/cpp/greedy_causal_lm/greedy_causal_lm ./redpajama-3b-chat/ "Alan Turing was a" > ./pred_greedy.txt
      - name: Compare
        run: |
          python -c "
          import transformers
          with open('pred_greedy.txt', 'r') as file:
              predictions = file.read()
          tokenizer = transformers.AutoTokenizer.from_pretrained('ikala/redpajama-3b-chat')
          tokenized = tokenizer('Alan Turing was a', return_tensors='pt')
          for output in transformers.AutoModelForCausalLM.from_pretrained('ikala/redpajama-3b-chat').generate(**tokenized, max_length=100, do_sample=False):
              ref = tokenizer.decode(output[tokenized['input_ids'].numel():], skip_special_tokens=True)
              idx = predictions.find(ref)
              if -1 == idx:
                  raise RuntimeError(f'Missing "{ref}" from predictions')
              predictions = predictions[:idx] + predictions[idx + len(ref):]
          "
          echo "Alan Turing was a" passed
      - run: >
          . ./ov/setupvars.sh
          && export PYTHONPATH=./build/:$PYTHONPATH
          && timeout 50s samples/python/greedy_causal_lm/greedy_causal_lm.py ./redpajama-3b-chat/ "Alan Turing was a"
          | diff ./pred_greedy.txt -