need additional_special_tokens argument for HFLM initializer

lm_eval/evaluator.py

@@ -69,6 +69,7 @@ def simple_evaluate(
         assert isinstance(model, lm_eval.base.LM)
         lm = model
 
+    print(f"evaluator: no_cache={no_cache}")
     if not no_cache:
         lm = lm_eval.base.CachingLM(
             lm,

lm_eval/models/gpt2.py

@@ -20,6 +20,7 @@ def __init__(
         load_in_8bit: Optional[bool] = False,
         trust_remote_code: Optional[bool] = False,
         use_fast: Optional[bool] = True,
+        additional_special_tokens: Optional[str] = None
     ):
         super().__init__()
 
@@ -66,6 +67,7 @@ def __init__(
             revision=revision,
             trust_remote_code=trust_remote_code,
             use_fast=use_fast,
+            additional_special_tokens=additional_special_tokens
         )
         self.vocab_size = self.tokenizer.vocab_size
 

main.py

@@ -88,6 +88,7 @@ def main():
     if args.description_dict_path:
         with open(args.description_dict_path, "r") as f:
             description_dict = json.load(f)
+    print(f"main: no_cache={args.no_cache}")
     results = evaluator.simple_evaluate(
         model=args.model,
         model_args=args.model_args,

models/stabilityai/stabilityai-japanese-stablelm-base-alpha-7b/harness.jcqa-1.2.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 set -eu
-MODEL_ARGS="pretrained=stabilityai/japanese-stablelm-base-alpha-7b,use_fast=False,trust_remote_code=True,device_map=auto,torch_dtype=auto,offload_folder=/tmp"
+MODEL_ARGS="pretrained=stabilityai/japanese-stablelm-base-alpha-7b,use_fast=False,trust_remote_code=True,device_map=auto,torch_dtype=auto,load_in_8bit=True,offload_folder=/tmp,tokenizer=novelai/nerdstash-tokenizer-v1,additional_special_tokens=['▁▁']"
 TASK="jcommonsenseqa-1.2-0.2"
 NUM_FEW_SHOTS="3"
 python main.py \
@@ -9,4 +9,5 @@ python main.py \
     --tasks $TASK \
     --num_fewshot $NUM_FEW_SHOTS \
     --device "cuda" \
+    --no_cache \
     --output_path "models/stablelm/stablelm-ja-base-alpha-7b/result.jcqa-1.2.json"