From edc92615e1f4584743189a9f339b3205699947d3 Mon Sep 17 00:00:00 2001 From: George Perdrizet Date: Sat, 27 Jul 2024 14:30:01 -0400 Subject: [PATCH 1/3] Added fragment scoring logic to scoring function. --- api/configuration.py | 10 ++- api/functions/flask_app.py | 20 +++--- api/functions/helper.py | 43 +++++++++++ api/functions/scoring.py | 126 ++++++++++++++++++++++++++++++++- classifier/__main__.py | 2 +- classifier/functions/helper.py | 1 - 6 files changed, 186 insertions(+), 16 deletions(-) diff --git a/api/configuration.py b/api/configuration.py index 3c3c8da..868af8c 100644 --- a/api/configuration.py +++ b/api/configuration.py @@ -10,8 +10,8 @@ # Set mode to testing to mock scoring function with random output # between 0.0 and 1.0 and not load any LLMs. Set to production -# to run real scoring function -MODE = 'testing' +# to run real scoring function. +MODE = 'production' # Get path to this config file so that we can define # other paths relative to it @@ -19,6 +19,7 @@ # Other project paths LOG_PATH=f'{PROJECT_ROOT_PATH}/logs' +DATA_PATH=f'{PROJECT_ROOT_PATH}/data' # Logging stuff LOG_LEVEL='DEBUG' @@ -42,6 +43,11 @@ CALCULATION_DEVICE='cuda:0' +PERPLEXITY_RATIO_KLD_KDE = f'{DATA_PATH}/perplexity_ratio_KLD_KDE.pkl' +TFIDF_LUT = f'{DATA_PATH}/TFIDF_lut.pkl' +TFIDF_SCORE_KLD_KDE = f'{DATA_PATH}/TFIDF_score_KLD_KDE.pkl' +XGBOOST_CLASSIFIER = f'{DATA_PATH}/XGBoost_classifier.pkl' + ###################################################################### # NON-HF default model parameters #################################### ###################################################################### diff --git a/api/functions/flask_app.py b/api/functions/flask_app.py index ced0a48..786d381 100644 --- a/api/functions/flask_app.py +++ b/api/functions/flask_app.py @@ -21,10 +21,10 @@ def __call__(self, *args: object, **kwargs: object) -> object: return self.run(*args, **kwargs) # Create Celery app - celery_app = Celery(app.name, task_cls=FlaskTask) + celery_app = Celery(app.name, task_cls = FlaskTask) # Add configuration from Flask app's Celery config. dict - celery_app.config_from_object(app.config["CELERY"]) + celery_app.config_from_object(app.config['CELERY']) # Configure logging celery_app.log.setup( @@ -35,7 +35,7 @@ def __call__(self, *args: object, **kwargs: object) -> object: # Set as default and add to extensions celery_app.set_default() - app.extensions["celery"] = celery_app + app.extensions['celery'] = celery_app return celery_app @@ -80,6 +80,13 @@ def score_text(suspect_string: str) -> str: # Mock the score with a random float score = [random.uniform(0, 1)] + # Threshold the score + if score[0] >= 0.5: + call = 'human' + + elif score[0] < 0.5: + call = 'synthetic' + elif config.MODE == 'production': # Call the scoring function @@ -89,12 +96,7 @@ def score_text(suspect_string: str) -> str: suspect_string ) - # Threshold the score - if score[0] >= 0.5: - call = 'human' - - elif score[0] < 0.5: - call = 'synthetic' + call = f'Prediction result: {score}' # Return the result from the output queue return {'author_call': call, 'text': suspect_string} diff --git a/api/functions/helper.py b/api/functions/helper.py index 1057806..edf862f 100644 --- a/api/functions/helper.py +++ b/api/functions/helper.py @@ -2,8 +2,10 @@ from __future__ import annotations from typing import Callable + import os import glob +import re import logging from threading import Thread from logging.handlers import RotatingFileHandler @@ -100,3 +102,44 @@ def start_flask(flask_app: Callable, logger: Callable): # Start the flask app thread flask_app_thread.start() + +def clean_text(text: str = None, sw = None, lemmatizer = None) -> str: + '''Cleans up text string for TF-IDF''' + + # Lowercase everything + text = text.lower() + + # Replace everything with space except (a-z, A-Z, ".", "?", "!", ",") + text = re.sub(r"[^a-zA-Z?.!,¿]+", " ", text) + + # Remove URLs + text = re.sub(r"http\S+", "",text) + + # Remove html tags + html = re.compile(r'<.*?>') + text = html.sub(r'',text) + + punctuations = '@#!?+&*[]-%.:/();$=><|{}^' + "'`" + '_' + + # Remove punctuations + for p in punctuations: + text = text.replace(p,'') + + # Remove stopwords + text = [word.lower() for word in text.split() if word.lower() not in sw] + text = [lemmatizer.lemmatize(word) for word in text] + text = " ".join(text) + + # Remove emojis + emoji_pattern = re.compile("[" + u"\U0001F600-\U0001F64F" # emoticons + u"\U0001F300-\U0001F5FF" # symbols & pictographs + u"\U0001F680-\U0001F6FF" # transport & map symbols + u"\U0001F1E0-\U0001F1FF" # flags (iOS) + u"\U00002702-\U000027B0" + u"\U000024C2-\U0001F251" + "]+", flags=re.UNICODE) + + text = emoji_pattern.sub(r'', text) + + return text \ No newline at end of file diff --git a/api/functions/scoring.py b/api/functions/scoring.py index 95f5fd2..b523f0c 100644 --- a/api/functions/scoring.py +++ b/api/functions/scoring.py @@ -1,9 +1,14 @@ '''Collection of functions to score strings''' from typing import Callable +import pickle import numpy as np import torch import transformers +from nltk.corpus import stopwords +from nltk.stem import WordNetLemmatizer + +import api.functions.helper as helper_funcs import api.configuration as config def score_string( @@ -14,6 +19,27 @@ def score_string( '''Takes a string, computes and returns llm detector score''' + # To run the XGBoost classifier, we need the following 9 features for + # this fragment: + feature_names = [ + 'Fragment length (tokens)', + 'Perplexity', + 'Cross-perplexity', + 'Perplexity ratio score', + 'Perplexity ratio Kullback-Leibler score', + 'Human TF-IDF', + 'Synthetic TF-IDF', + 'TF-IDF score', + 'TF-IDF Kullback-Leibler score' + ] + + # Empty holder for features + features = [] + + ############################################################### + # Get perplexity, cross-perplexity and perplexity ratio score # + ############################################################### + # Encode the string using the reader's tokenizer encodings = reader_model.tokenizer( string, @@ -21,14 +47,19 @@ def score_string( return_token_type_ids = False ).to(reader_model.device_map) + # Get the string length in tokens and add to features + fragment_length = encodings['input_ids'].shape[1] + features.append(fragment_length) + # Calculate logits reader_logits = reader_model.model(**encodings).logits writer_logits = writer_model.model(**encodings).logits - # Calculate perplexity + # Calculate perplexity and add to features ppl = perplexity(encodings, writer_logits) + features.append(ppl[0]) - # Calculate cross perplexity + # Calculate cross perplexity and add to features x_ppl = entropy( reader_logits.to(config.CALCULATION_DEVICE), writer_logits.to(config.CALCULATION_DEVICE), @@ -36,10 +67,99 @@ def score_string( reader_model.tokenizer.pad_token_id ) + features.append(x_ppl[0]) + + # Calculate perplexity ratio and add to features scores = ppl / x_ppl scores = scores.tolist() + perplexity_ratio_score = scores[0] + features.append(perplexity_ratio_score) + + ############################################################### + # Get perplexity ratio Kullback-Liebler score ################# + ############################################################### + + # Load the perplexity ratio Kullback-Leibler kernel density estimate + with open(config.PERPLEXITY_RATIO_KLD_KDE, 'rb') as input_file: + perplexity_ratio_kld_kde = pickle.load(input_file) + + # Calculate perplexity ratio KLD score and add to features + perplexity_ratio_kld_score = perplexity_ratio_kld_kde.pdf(perplexity_ratio_score) + features.append(perplexity_ratio_kld_score[0]) + + ############################################################### + # Get human and synthetic TF-IDFs and TF-IDF score ############ + ############################################################### + + # Load the TF-IDF luts + with open(config.TFIDF_LUT, 'rb') as input_file: + tfidf_luts = pickle.load(input_file) + + # Clean the test for TF-IDF scoring + sw = stopwords.words('english') + lemmatizer = WordNetLemmatizer() + + cleaned_string = helper_funcs.clean_text( + text = string, + sw = sw, + lemmatizer = lemmatizer + ) + + # Split cleaned string into words + words = cleaned_string.split(' ') + + # Initialize TF-IDF sums + human_tfidf_sum = 0 + synthetic_tfidf_sum = 0 + + # Score the words using the human and synthetic luts + for word in words: + + if word in tfidf_luts['human'].keys(): + human_tfidf_sum += tfidf_luts['human'][word] + + if word in tfidf_luts['synthetic'].keys(): + synthetic_tfidf_sum += tfidf_luts['synthetic'][word] + + # Get the means and add to features + human_tfidf_mean = human_tfidf_sum / len(words) + synthetic_tfidf_mean = synthetic_tfidf_sum / len(words) + dmean_tfidf = human_tfidf_mean - synthetic_tfidf_mean + product_normalized_dmean_tfidf = dmean_tfidf * (human_tfidf_mean + synthetic_tfidf_mean) + + features.append(human_tfidf_mean) + features.append(synthetic_tfidf_mean) + features.append(product_normalized_dmean_tfidf) + + ############################################################### + # Get TF-IDF Kullback-Liebler score ########################### + ############################################################### + + # Load the TF_IDF Kullback-Leibler kernel density estimate + with open(config.TFIDF_SCORE_KLD_KDE, 'rb') as input_file: + tfidf_kld_kde = pickle.load(input_file) + + # Calculate TF-IDF LKD score and add to features + tfidf_kld_score = tfidf_kld_kde.pdf(product_normalized_dmean_tfidf) + features.append(tfidf_kld_score[0]) + + print(f'Features complete:') + + for feature_name, feature_value in zip(feature_names, features): + print(f'{feature_name}: {feature_value}') + + ############################################################### + # Run infrence with the classifier ############################ + ############################################################### + + # Load the model + with open(config.XGBOOST_CLASSIFIER, 'rb') as input_file: + model = pickle.load(input_file) + + # Make prediction + prediction = model.predict([features]) - return scores + return prediction # Take some care with '.sum(1)).detach().cpu().float().numpy()'. Had # errors as cribbed from the above repo. Order matters? I don't know, diff --git a/classifier/__main__.py b/classifier/__main__.py index 9d90d75..9186c2a 100644 --- a/classifier/__main__.py +++ b/classifier/__main__.py @@ -111,7 +111,7 @@ def run(self): if __name__ == '__main__': - helper_funcs.force_after('AddPerplexityRatioKLDScore') + helper_funcs.force_after('AddTFIDFScore') luigi.build( [ diff --git a/classifier/functions/helper.py b/classifier/functions/helper.py index 8dc7e1c..f0f4556 100644 --- a/classifier/functions/helper.py +++ b/classifier/functions/helper.py @@ -225,7 +225,6 @@ def tfidf_score_text_fragments(data_chunk: pd.DataFrame, tfidf_luts: dict = None human_tfidf_means.append(human_tfidf_mean) synthetic_tfidf_means.append(synthetic_tfidf_mean) - dmean_tfidfs.append(dmean_tfidf) product_normalized_dmean_tfidfs.append(product_normalized_dmean_tfidf) data_chunk['Human TF-IDF'] = human_tfidf_means From c6fd1b09f6c3103e6a4e21c4cf8ddb7353ca2751 Mon Sep 17 00:00:00 2001 From: George Perdrizet Date: Sat, 27 Jul 2024 21:03:40 -0400 Subject: [PATCH 2/3] Moved all response generation and formatting out of telegram bot and into flask app. --- api/functions/flask_app.py | 10 +++++----- telegram_bot/__main__.py | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/api/functions/flask_app.py b/api/functions/flask_app.py index 786d381..2e64cdc 100644 --- a/api/functions/flask_app.py +++ b/api/functions/flask_app.py @@ -114,9 +114,9 @@ def submit_text() -> dict: # Submit the text for scoring result = score_text.delay(text_string) - return {"result_id": result.id} + return {'result_id': result.id} - @app.get("/result/") + @app.get('/result/') def task_result(result_id: str) -> dict: '''Gets result by result id. Returns dictionary with task status''' @@ -126,9 +126,9 @@ def task_result(result_id: str) -> dict: # Return status and result if ready return { - "ready": result.ready(), - "successful": result.successful(), - "value": result.result if result.ready() else None, + 'ready': result.ready(), + 'successful': result.successful(), + 'value': result.result if result.ready() else None, } return app diff --git a/telegram_bot/__main__.py b/telegram_bot/__main__.py index df123cb..63c9ada 100644 --- a/telegram_bot/__main__.py +++ b/telegram_bot/__main__.py @@ -17,7 +17,8 @@ async def start(update: Update, context: ContextTypes.DEFAULT_TYPE): async def score_text(update: Update, context: ContextTypes.DEFAULT_TYPE): - '''Sends user provided text to scoring function''' + '''Sends user provided text to scoring function, sends + result back to user.''' # Get the logger logger = logging.getLogger(f'telegram_bot.score_text') @@ -31,15 +32,14 @@ async def score_text(update: Update, context: ContextTypes.DEFAULT_TYPE): # Get the result, when ready result = api_funcs.retreive_result(result_id = result_id) - author_call = await result - reply = f'Author is likley {author_call}' + reply = await result logger.info(f'Got user text: {text}') logger.info(f'Result ID: {result_id}') logger.info(f'Reply: {reply}') await context.bot.send_message( - chat_id=update.effective_chat.id, text=reply) + chat_id = update.effective_chat.id, text=reply) if __name__ == '__main__': From f87858baeb1d1368d479da5029a72b59f14caf74 Mon Sep 17 00:00:00 2001 From: George Perdrizet Date: Sat, 27 Jul 2024 21:07:47 -0400 Subject: [PATCH 3/3] Added logic to call author is human or synthetic based on XGBoost binary output. --- api/functions/flask_app.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/api/functions/flask_app.py b/api/functions/flask_app.py index 2e64cdc..7241b98 100644 --- a/api/functions/flask_app.py +++ b/api/functions/flask_app.py @@ -96,10 +96,16 @@ def score_text(suspect_string: str) -> str: suspect_string ) - call = f'Prediction result: {score}' + if score[0] == 0: + call = 'human' + + elif score[0] == 1: + call = 'synthetic' + + reply = f'Text is likley {call}.' # Return the result from the output queue - return {'author_call': call, 'text': suspect_string} + return {'author_call': reply, 'text': suspect_string} # Set listener for text strings via POST @app.post('/submit_text')