gpt-3.5-data-generation-trans.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8de529e8-3891-4f47-8585-65b92b80bbf7",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c0328f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "identity = \"trans\"\n",
    "\n",
    "output_file = f\"gpt3.5_generated_hs_dataset_{identity}.json\"\n",
    "print(output_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e23399a9",
   "metadata": {
    "height": 132
   },
   "outputs": [],
   "source": [
    "import openai\n",
    "import os\n",
    "\n",
    "from dotenv import load_dotenv, find_dotenv\n",
    "_ = load_dotenv(find_dotenv()) # read local .env file\n",
    "\n",
    "openai.api_key  = os.getenv('OPENAI_API_KEY')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c943900-98db-40b3-b639-7822ee2c5fd8",
   "metadata": {
    "height": 161
   },
   "outputs": [],
   "source": [
    "def get_completion(prompt, model=\"gpt-3.5-turbo\",temperature=0): # Andrew mentioned that the prompt/ completion paradigm is preferable for this class\n",
    "    messages = [{\"role\": \"system\", \"content\": system_message}, \n",
    "                {\"role\": \"user\", \"content\": prompt}]\n",
    "    response = openai.ChatCompletion.create(\n",
    "        model=model,\n",
    "        messages=messages,\n",
    "        temperature=temperature, # this is the degree of randomness of the model's output\n",
    "    )\n",
    "    return response.choices[0].message[\"content\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39a37f41",
   "metadata": {
    "height": 212
   },
   "outputs": [],
   "source": [
    "# helper constant and methods to prepare the prompts\n",
    "\n",
    "system_message = \"\"\"You are a crowd-source worker who composes messages for a hate speech \n",
    "                    project. Write messages with diverse style and length.\"\"\"\n",
    "\n",
    "def prompt_factory(msg, n_examples=10, demonstration=None):\n",
    "    if demonstration:\n",
    "        return f\"\"\"\n",
    "            Write {n_examples} messages that {msg}.\n",
    "            Example: ```{demonstration}```\n",
    "            Return the result in JSON list format with the following key: messages.\n",
    "            Result:\n",
    "            \"\"\"\n",
    "    else:\n",
    "        return f\"\"\"\n",
    "            Write {n_examples} messages that {msg}.\n",
    "            Return the result in JSON list format with the following key: messages.\n",
    "            Result:\n",
    "            \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5403f73",
   "metadata": {
    "height": 302
   },
   "outputs": [],
   "source": [
    "import json\n",
    "from math import ceil\n",
    "import logging\n",
    "\n",
    "def get_messages(prompt_phrase, demonstration=None, n=40, temperature=0.5):\n",
    "    if not demonstration:\n",
    "        prompt = prompt_factory(prompt_phrase, n_examples=20)\n",
    "    else:\n",
    "        prompt = prompt_factory(prompt_phrase, demonstration=demonstration, n_examples=20)\n",
    "    \n",
    "    print(prompt)\n",
    "    result = list()\n",
    "    num_calls = 0\n",
    "    while len(result) < n:\n",
    "        response = get_completion(prompt, temperature=temperature)\n",
    "        try:\n",
    "            response_json = json.loads(response)\n",
    "            result.extend(response_json['messages'])\n",
    "        except Exception:\n",
    "            logging.exception(\"An exception was thrown!\")\n",
    "        print(f\"num_calls: {num_calls}; num_messages: {len(result)}\")\n",
    "        num_calls += 1\n",
    "    return [prompt_phrase]*len(result), result"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bdb008d3",
   "metadata": {},
   "source": [
    "## Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5446198f",
   "metadata": {
    "height": 59
   },
   "outputs": [],
   "source": [
    "# initialize result. IMPORTANT: run only once, otherwise it'll be erased\n",
    "prompts, messages = list(), list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa0dd40a",
   "metadata": {
    "height": 212
   },
   "outputs": [],
   "source": [
    "# F1: Expression of strong negative emotions (explicit)\n",
    "prompt_phrase = f\"\"\"express strong negative emotions against {identity}\"\"\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "687877df",
   "metadata": {
    "height": 212
   },
   "outputs": [],
   "source": [
    "# F2: Description using very negative attributes (explicit)\n",
    "prompt_phrase = f\"\"\"mention very negative attributes of {identity}\"\"\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8738632d",
   "metadata": {
    "height": 212
   },
   "outputs": [],
   "source": [
    "# F3: Dehumanisation (explicit)\n",
    "prompt_phrase = f\"\"\"dehumanize {identity}\"\"\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c94690b1",
   "metadata": {
    "height": 200
   },
   "outputs": [],
   "source": [
    "# F4: Implicit derogation\n",
    "prompt_phrase = f\"\"\"implicitly derogates {identity}\"\"\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f40523ec",
   "metadata": {
    "height": 200
   },
   "outputs": [],
   "source": [
    "# F5: Direct threat\n",
    "prompt_phrase = f\"\"\"contain direct threat to {identity} mentioning their identity\"\"\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5e805db",
   "metadata": {
    "height": 212
   },
   "outputs": [],
   "source": [
    "# F6: Threat as normative statement\n",
    "prompt_phrase = f\"\"\"contain threat to {identity} as a normative statement\"\"\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9216686b",
   "metadata": {
    "height": 212
   },
   "outputs": [],
   "source": [
    "# F7: Hate expressed using slur\n",
    "prompt_phrase = f\"\"\"contain hate expressions against {identity} using slurs\"\"\"\n",
    "example = \"You are nothing but a tranny.\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase, demonstration=example)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a4c98f0",
   "metadata": {
    "height": 212
   },
   "outputs": [],
   "source": [
    "# F8: Non-hateful homonyms of slurs\n",
    "prompt_phrase = f\"\"\"contain non-hateful homonyms of slurs of {identity}\"\"\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "287e528c",
   "metadata": {
    "height": 234
   },
   "outputs": [],
   "source": [
    "# F9: Reclaimed slurs\n",
    "prompt_phrase = f\"\"\"reclaim one of the following slurs referring to {identity} to show endearment in a non-hateful context.\n",
    "Slurs: ```trans, tranny, trannies, shemale, shemales```\"\"\"\n",
    "example = \"I'm the prettiest trans in the room.\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase, demonstration=example)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46c94856",
   "metadata": {
    "height": 212
   },
   "outputs": [],
   "source": [
    "# F10: Hate expressed using profanity\n",
    "prompt_phrase = f\"\"\"contain hate expressions against {identity} using profanity\"\"\"\n",
    "example = f\"All {identity} should fuck off forever.\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase, demonstration=example)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5447f113",
   "metadata": {
    "height": 212
   },
   "outputs": [],
   "source": [
    "# F11: Non-hateful use of profanity\n",
    "# Omitted. non identity-specific"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fb25495",
   "metadata": {
    "height": 280
   },
   "outputs": [],
   "source": [
    "# F12: Hate expressed through reference in subsequent clauses\n",
    "prompt_phrase = f\"\"\"\n",
    "1. each contain two clauses\n",
    "2. the second clause contains hate expressions against {identity}\"\"\"\n",
    "example = f\"Whenever {identity} look at me, I think they can sense that I feel nothing but contempt for them.\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase, demonstration=example)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb3a5fe0",
   "metadata": {
    "height": 285
   },
   "outputs": [],
   "source": [
    "# F13: Hate expressed through reference in subsequent sentences\n",
    "prompt_phrase = f\"\"\"\n",
    "1. each contain two sentenses\n",
    "2. the second sentence contains hate expressions against {identity}\"\"\"\n",
    "example = f\"I know {identity} like you can't help it. You're just all that dumb.\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase, demonstration=example)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20c8e794",
   "metadata": {
    "height": 246
   },
   "outputs": [],
   "source": [
    "# F14: Hate expressed using negated positive statement\n",
    "prompt_phrase = f\"\"\"contain hate expressed against {identity} by negating a positive statement\"\"\"\n",
    "example = f\"{identity} are never any intelligent.\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase, demonstration=example)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6fd645e",
   "metadata": {
    "height": 246
   },
   "outputs": [],
   "source": [
    "# F15: Non-hate expressed using negated hateful statement\n",
    "prompt_phrase = f\"\"\"are not offensive and mention {identity} in a negated hateful statement\"\"\"\n",
    "example = f\"No {identity} deserves to die.\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase, demonstration=example)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fde5863d",
   "metadata": {
    "height": 200
   },
   "outputs": [],
   "source": [
    "# F16: Hate phrased as a question\n",
    "prompt_phrase = f\"\"\"contain hate phrases against {identity} as a question\"\"\"\n",
    "example= f\"Do people really not realise that {identity} are disgusting?\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase, demonstration=example)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2f26afb",
   "metadata": {
    "height": 200
   },
   "outputs": [],
   "source": [
    "# F17: Hate phrased as an opinion\n",
    "prompt_phrase = f\"\"\"contain hate phrases against {identity} as an opinion\"\"\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "101c023d",
   "metadata": {
    "height": 200
   },
   "outputs": [],
   "source": [
    "# F18: Neutral statements using protected group identifier\n",
    "prompt_phrase = f\"\"\"contain factual statements about {identity} with neither positive or negative sentiment\"\"\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c901c821",
   "metadata": {
    "height": 200
   },
   "outputs": [],
   "source": [
    "# F19: Positive statements using pro- tected group identifier\n",
    "prompt_phrase = f\"\"\"contain positive statements about {identity}\"\"\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c68f6f3b",
   "metadata": {
    "height": 234
   },
   "outputs": [],
   "source": [
    "# F20: Denouncements of hate that quote it\n",
    "prompt_phrase = f\"\"\"\n",
    "1. mention hateful statements against {identity} in quote\n",
    "2. denounce the statement\"\"\"\n",
    "example = f\"Statements like ‘{identity} are scum’ are deeply hurtful.\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase, demonstration=example)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4fe11d66",
   "metadata": {
    "height": 246
   },
   "outputs": [],
   "source": [
    "# F21: Denouncements of hate that make direct reference to it\n",
    "prompt_phrase = f\"\"\"mention a hateful statement made against {identity} and denounce it. \n",
    "Make sure your response does not contain generic words like 'hate', 'hateful', or 'derogatory'\"\"\"\n",
    "example = f\"\"\"I lost all respect for you when you said {identity} like me were pathetic.\"\"\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase, demonstration=example)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4175c20",
   "metadata": {
    "height": 200
   },
   "outputs": [],
   "source": [
    "# F22: Abuse targeted at objects\n",
    "# Omitted since not targeting an identity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ede2cf6a",
   "metadata": {
    "height": 200
   },
   "outputs": [],
   "source": [
    "# F23: Abuse targeted at individuals (not as member of a prot. group)\n",
    "# Omitted since not targeting an identity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b9ec683",
   "metadata": {
    "height": 234
   },
   "outputs": [],
   "source": [
    "# F24: Abuse targeted at non-protected groups (e.g. professions)\n",
    "# Omitted since not targeting an identity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c38b751",
   "metadata": {
    "height": 246
   },
   "outputs": [],
   "source": [
    "# F25-29: Spelling variation\n",
    "prompt_phrase = f\"\"\"contain hateful statements against {identity} with different kinds of misspellings\"\"\"\n",
    "example = f\"No one else is as f1lthy as theez {identity}.\"\n",
    "\n",
    "batch_prompts, batch_messages = get_messages(prompt_phrase, demonstration=example)\n",
    "prompts.extend(batch_prompts)\n",
    "messages.extend(batch_messages)\n",
    "\n",
    "# save after each prompt in case the instance got shut down\n",
    "final_json = {\"prompt\": prompts, \"message\": messages}\n",
    "with open(output_file, \"w+\") as f:\n",
    "    f.write(f\"{final_json}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d98736d2",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "len(messages), len(prompts)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}