add all files for release v0.1.0

.gitignore

@@ -0,0 +1,3 @@
+build/
+selfcheckgpt.egg-info
+examples/

DESCRIPTION.txt

@@ -0,0 +1,36 @@
+SelfCheckGPT
+=====================================================
+
+- Code for our paper "[SelfCheckGPT: Zero-Resource Black-Box Hallucination Detection for Generative Large Language Models](https://arxiv.org/abs/2303.08896)"
+- More information can be found on our [project page](https://github.com/potsawee/selfcheckgpt)
+
+Installation
+-----------------------------------------------------
+
+    pip install selfcheckgpt
+
+
+Example
+-----------------------------------------------------
+
+```python
+from selfcheckgpt.modeling_mqag import SelfCheckMQAG
+from selfcheckgpt.modeling_bertscore import SelfCheckBERTScore
+
+selfcheck_mqag = SelfCheckMQAG()
+selfcheck_bertscore = SelfCheckBERTScore()
+
+sent_scores_mqag = selfcheck_mqag.predict(
+    sentences,
+    passage,
+    [sample1, sample2, sample3],
+    num_questions_per_sent = 5,
+    scoring_method = 'bayes_with_alpha',
+    beta1 = 0.8, beta2 = 0.8,
+)
+sent_scores_bertscore = selfcheck_bertscore.predict(
+    sentences,
+    [sample1, sample2, sample3],
+)
+
+```

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Potsawee Manakul
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -2,11 +2,37 @@ SelfCheckGPT
 =====================================================
 Project page for our paper "[SelfCheckGPT: Zero-Resource Black-Box Hallucination Detection for Generative Large Language Models](https://arxiv.org/abs/2303.08896)"
 
-## Code/Models
+## Code/Package
 
-**Code is to be updated soon**
+### Installation
 
-## Dataset 
+    pip install selfcheckgpt
+
+### Example
+
+```python
+from selfcheckgpt.modeling_mqag import SelfCheckMQAG
+from selfcheckgpt.modeling_bertscore import SelfCheckBERTScore
+
+selfcheck_mqag = SelfCheckMQAG()
+selfcheck_bertscore = SelfCheckBERTScore()
+
+sent_scores_mqag = selfcheck_mqag.predict(
+    sentences,
+    passage,
+    [sample1, sample2, sample3],
+    num_questions_per_sent = 5,
+    scoring_method = 'bayes_with_alpha',
+    beta1 = 0.8, beta2 = 0.8,
+)
+sent_scores_bertscore = selfcheck_bertscore.predict(
+    sentences,
+    [sample1, sample2, sample3],
+)
+
+```
+
+## Dataset
 The `wiki_bio_gpt3_hallucination` dataset currently consists of 65 annotated passages. You can find more information in the paper or our data card on HuggingFace: https://huggingface.co/datasets/potsawee/wiki_bio_gpt3_hallucination. To use this dataset, you can either load it through HuggingFace dataset API, or download it directly from below in the JSON format.
 
 ### Option1: HuggingFace
@@ -37,13 +63,11 @@ Each instance consists of:
 
 ```
 @misc{manakul2023selfcheckgpt,
-      title={SelfCheckGPT: Zero-Resource Black-Box Hallucination Detection for Generative Large Language Models}, 
+      title={SelfCheckGPT: Zero-Resource Black-Box Hallucination Detection for Generative Large Language Models},
       author={Potsawee Manakul and Adian Liusie and Mark J. F. Gales},
       year={2023},
       eprint={2303.08896},
       archivePrefix={arXiv},
       primaryClass={cs.CL}
 }
 ```
-
-

demo/demo1.py

@@ -0,0 +1,54 @@
+import torch
+import spacy
+from selfcheckgpt.modeling_mqag import SelfCheckMQAG
+from selfcheckgpt.modeling_bertscore import SelfCheckBERTScore
+
+passage = """
+Michael Alan Weiner (born March 31, 1942), better known by his professional name Michael Savage, is an American radio host, author, activist, nutritionist, and conservative political commentator. He is the host of The Savage Nation, a nationally syndicated talk show that aired on Talk Radio Network across the United States until 2012, and in 2009 was the second most listened-to radio talk show in the country with an audience of over 20 million listeners on 400 stations across the United States. Since October 23, 2012, Michael Savage has been syndicated by Cumulus Media Networks. He holds master's degrees from the University of Hawaii in medical botany and medical anthropology, and a Ph.D. from the University of California, Berkeley in nutritional ethnomedicine. As Michael Weiner, he has written books on nutrition, herbal medicine, and homeopathy.
+""".replace("\n", " ").strip()
+
+sample1 = """
+Michael Alan Weiner (born March 31, 1942), better known by his professional name Michael Savage, is an American radio host, author, and political commentator. He is the host of The Savage Nation, a nationally syndicated talk show broadcast on Talk Radio Network across the United States, and on Comet in Canada. He holds master's degrees from the University of Hawaii in medical botany and medical anthropology, and a Ph.D. from the University of California, Berkeley in epidemiology and nutrition sciences. Savage has summarized his political philosophy in three words: borders, language, and culture. He has characterized his views as conservative nationalism, while critics have characterized them as fascism. He is a fierce critic of liberalism, and the War on Drugs. He believes that political correctness has "ruined the United States" and that the American government has spent too much money on social programs throughout its history.
+""".replace("\n", " ").strip()
+
+sample2 = """
+Michael A. Savage (born Michael Alan Weiner; March 31, 1942) is an American radio host, author, activist, nutritionist, and conservative political commentator. He is the host of The Savage Nation, a nationally syndicated talk show that aired on Talk Radio Network across the United States until 2012. The Savage Nation has a combined audience of over 20 million listeners on 400 stations across the United States and is one of the top-rated shows in the country. He is also the host of The Michael Savage Show, a daily podcast and webcast heard around the world. He has been called "the godfather of Trumpmania".
+
+Savage has written more than 25 books and is an outspoken supporter of traditional American values and break-down of the culture of political correctness. He advocates an "America first" foreign policy and rigorous opposition to illegal immigration. For his social and political views, Savage has been described as a conservative and a right-wing populist. He has campaigned for numerous political causes, including opposition to same-sex marriage and the legalization of marijuana.
+""".replace("\n", " ").strip()
+
+sample3 = """
+Michael Alan Weiner (born March 31, 1942),[2] better known by his professional name Michael Savage, is an American radio host, author, activist, nutritionist, and conservative political commentator. He is the host of The Savage Nation, a nationally syndicated talk show that aired on Talk Radio Network across the United States until 2012, and in 2009 was the second most listened-to radio talk show in the country with an audience of over 20 million listeners on 400 stations across the United States. Since October 23, 2012, Michael Savage has been syndicated by Cumulus Media Networks. He holds master's degrees from the University of Hawaii in medical botany and medical anthropology, and a Ph.D. from the University of California, Berkeley in nutritional ethnomedicine. As Michael Weiner, he has written books on nutrition, herbal medicine, and homeopathy.
+
+Savage has summarized his political philosophy in three words: borders, language, and culture. He believes that the United States should end foreign aid, impose much stricter immigration laws, deport illegal immigrants, and put an end to birthright citizenship. Savage has characterized his views as conservative nationalism, free
+""".replace("\n", " ").strip()
+
+def experiment():
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    nlp = spacy.load("en_core_web_sm")
+    sentences = [sent for sent in nlp(passage).sents] # List[spacy.tokens.span.Span]
+    sentences = [sent.text.strip() for sent in sentences if len(sent) > 3]
+    selfcheck_mqag = SelfCheckMQAG(device)
+    selfcheck_bertscore = SelfCheckBERTScore()
+    print("SelfCheck running on {} sentences...".format(len(sentences)))
+    sent_scores_mqag = selfcheck_mqag.predict(
+        sentences,
+        passage,
+        [sample1, sample2, sample3],
+        num_questions_per_sent = 5,
+        scoring_method = 'bayes_with_alpha',
+        beta1 = 0.8, beta2 = 0.8,
+    )
+    sent_scores_bertscore = selfcheck_bertscore.predict(
+        sentences,
+        [sample1, sample2, sample3],
+    )
+    print("---------------------------------")
+    print("MQAG\tBERTScore")
+    for s1, s2 in zip(sent_scores_mqag, sent_scores_bertscore):
+        print("{:.4f}\t{:.4f}".format(s1, s2))
+    print("---------------------------------")
+
+
+if __name__ == "__main__":
+    experiment()

selfcheckgpt/modeling_bertscore.py

@@ -0,0 +1,56 @@
+import spacy
+import bert_score
+import numpy as np
+import torch
+from typing import Dict, List, Set, Tuple, Union
+from transformers import logging
+logging.set_verbosity_error()
+
+def expand_list1(mylist, num):
+    expanded = []
+    for x in mylist:
+        for _ in range(num):
+            expanded.append(x)
+    return expanded
+
+def expand_list2(mylist, num):
+    expanded = []
+    for _ in range(num):
+        for x in mylist:
+            expanded.append(x)
+    return expanded
+
+class SelfCheckBERTScore:
+    def __init__(self, default_model="en"):
+        self.nlp = spacy.load("en_core_web_sm")
+        self.default_model = default_model # en => roberta-large
+        print("SelfCheck-BERTScore initialized")
+
+    @torch.no_grad()
+    def predict(
+            self,
+            sentences: List[str],
+            sampled_passages: List[str],
+        ):
+        num_sentences = len(sentences)
+        num_samples = len(sampled_passages)
+        bertscore_array = np.zeros((num_sentences, num_samples))
+        for s in range(num_samples):
+            sample_passage = sampled_passages[s]
+            sentences_sample = [sent for sent in self.nlp(sample_passage).sents] # List[spacy.tokens.span.Span]
+            sentences_sample = [sent.text.strip() for sent in sentences_sample if len(sent) > 3]
+            num_sentences_sample  = len(sentences_sample)
+
+            refs  = expand_list1(sentences, num_sentences_sample) # r1,r1,r1,....
+            cands = expand_list2(sentences_sample, num_sentences) # s1,s2,s3,...
+
+            P, R, F1 = bert_score.score(cands, refs, lang=self.default_model, verbose=False)
+            F1_arr = F1.reshape(num_sentences, num_sentences_sample)
+            F1_arr_max_axis1 = F1_arr.max(axis=1).values
+            F1_arr_max_axis1 = F1_arr_max_axis1.numpy()
+
+            bertscore_array[:,s] = F1_arr_max_axis1
+
+        bertscore_mean_per_sent = bertscore_array.mean(axis=-1)
+        one_minus_bertscore_mean_per_sent = 1.0 - bertscore_mean_per_sent
+        return one_minus_bertscore_mean_per_sent

selfcheck/selfcheck_mqag.py → selfcheckgpt/modeling_mqag.py

@@ -1,9 +1,10 @@
 import re
+from typing import Dict, List, Set, Tuple, Union
 import numpy as np
 import torch
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
 from transformers import LongformerTokenizer, LongformerForMultipleChoice, LongformerForSequenceClassification
-from selfcheck_utils import prepare_qa_input, prepare_distractor_input, prepare_answering_input
+from .utils import prepare_qa_input, prepare_distractor_input, prepare_answering_input
 
 def method_simple_counting(
         prob,
@@ -89,7 +90,7 @@ def method_bayes_with_alpha(
     return score
 
 class SelfCheckMQAG:
-    def __init__(self, device):
+    def __init__(self, device=None):
         # Question Generation Systems (G1 & G2)
         self.g1_tokenizer = AutoTokenizer.from_pretrained("potsawee/t5-large-generation-squad-QuestionAnswer")
         self.g1_model = AutoModelForSeq2SeqLM.from_pretrained("potsawee/t5-large-generation-squad-QuestionAnswer")
@@ -108,13 +109,17 @@ def __init__(self, device):
         self.g2_model.eval()
         self.a_model.eval()
         self.u_model.eval()
+
+        if device is None:
+            device = torch.device("cpu")
         self.g1_model.to(device)
         self.g2_model.to(device)
         self.a_model.to(device)
         self.u_model.to(device)
         self.device = device
-        print("SelfCheck-MQAG initialized")
+        print("SelfCheck-MQAG initialized to device", device)
 
+    @torch.no_grad()
     def predict(
             self,
             sentences: List[str],
@@ -156,7 +161,7 @@ def predict(
                 scores.append(score)
             sent_score = np.mean(scores)
             sent_scores.append(sent_score)
-        return sent_scores
+        return np.array(sent_scores)
 
     def _answerability_scoring(
             self,

setup.py

@@ -0,0 +1,38 @@
+from setuptools import setup
+
+# reading long description from file
+with open('DESCRIPTION.txt') as file:
+    long_description = file.read()
+
+REQUIREMENTS = [
+    "transformers>=4.11.3",
+    "torch>=1.10",
+    "numpy",
+    "bert_score",
+    "spacy",
+]
+
+# some more details
+CLASSIFIERS = [
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+]
+
+# calling the setup function
+setup(
+    name='selfcheckgpt',
+    version='0.1.0',
+    description='SelfCheckGPT - Assessing text-based responses from LLMs',
+    long_description=long_description,
+    url='https://github.com/potsawee/selfcheckgpt',
+    author='Potsawee Manakul',
+    author_email='m.potsawee@gmail.com',
+    license='MIT',
+    packages=['selfcheckgpt'],
+    classifiers=CLASSIFIERS,
+    install_requires=REQUIREMENTS,
+    keywords='selfcheckgpt'
+)