Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions convert-gpt4all-to-ggml.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
import glob
import os
import struct
import sys
from sentencepiece import SentencePieceProcessor

HPARAMS = keys = ["vocab_size", "dim", "multiple_of", "n_heads", "n_layers"]
Expand Down Expand Up @@ -56,7 +55,7 @@ def write_tokens(fout, tokenizer):
piece = tokenizer.id_to_piece(i)
if len(piece) != 6:
print(f"Invalid token: {piece}")
sys.exit(1)
raise SystemExit(1)
byte_value = int(piece[3:-1], 16)
text = struct.pack("B", byte_value)
else:
Expand Down
4 changes: 2 additions & 2 deletions convert-gptq-to-ggml.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

if len(sys.argv) != 4:
print("Usage: convert-gptq-to-ggml.py llamaXXb-4bit.pt tokenizer.model out.bin\n")
sys.exit(1)
raise SystemExit(1)

fname_model = sys.argv[1]
fname_tokenizer = sys.argv[2]
Expand Down Expand Up @@ -57,7 +57,7 @@
piece = tokenizer.id_to_piece(i)
if len(piece) != 6:
print(f"Invalid token: {piece}")
sys.exit(1)
raise SystemExit(1)
byte_value = int(piece[3:-1], 16)
text = struct.pack("B", byte_value)
else:
Expand Down
5 changes: 2 additions & 3 deletions convert-pth-to-ggml.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@

import argparse
import os
import sys
import json
import struct
import numpy as np
Expand Down Expand Up @@ -86,7 +85,7 @@ def get_n_parts(dim):
n_parts = mappings.get(dim)
if n_parts is None:
print(f"Invalid dim: {dim}")
sys.exit(1)
raise SystemExit(1)

print(f"n_parts = {n_parts}\n")
return n_parts
Expand Down Expand Up @@ -127,7 +126,7 @@ def write_tokens(fout, tokenizer):
piece = tokenizer.id_to_piece(i)
if len(piece) != 6:
print(f"Invalid token: {piece}")
sys.exit(1)
raise SystemExit(1)
byte_value = int(piece[3:-1], 16)
text = struct.pack("B", byte_value)
else:
Expand Down
3 changes: 1 addition & 2 deletions convert-unversioned-ggml-to-ggml.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import glob
import os
import struct
import sys
from sentencepiece import SentencePieceProcessor

HPARAMS = keys = ["vocab_size", "dim", "multiple_of", "n_heads", "n_layers"]
Expand Down Expand Up @@ -51,7 +50,7 @@ def write_tokens(fout, tokenizer):
piece = tokenizer.id_to_piece(i)
if len(piece) != 6:
print(f"Invalid token: {piece}")
sys.exit(1)
raise SystemExit(1)
byte_value = int(piece[3:-1], 16)
text = struct.pack("B", byte_value)
else:
Expand Down