process_raw_robustness_results.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This Notebook is meant for processing the raw moderation results that are to be processed in order to obtain the binary moderation outputs. Please provide the paths of the respective dictionaries stored as pickle files where each phrase in the datasets is mapped to the moderation output of the AI Safety Moderation Classifers (openai, clarifai, google perspective and google PaLM2)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "perturbations = ['german', 'gpt_3.5_turbo']\n",
    "gpt_moderations = [\n",
    "                    [\n",
    "                    './results/<openai_moderation_for_german_backtranslation>.pkl', # comments_backtranslated_german_moderated_openai\n",
    "                    './results/<google_perspective_moderation_for_german_backtranslation>.pkl', # comments_backtranslated_german_moderation_google_perspective\n",
    "                    './results/<google_palm_moderation_for_german_backtranslation>.pkl', # comments_backtranslated_german_moderation_google_latest_may25\n",
    "                    './results/<clarifai_moderation_for_german_backtranslation>.pkl' # german_comment_moderation_clarifai\n",
    "                    ],\n",
    "                    [\n",
    "                    './results/<openai_moderation_for_gpt_turbo_paraphrasing>.pkl', # comments_paraphrased_gpt-3.5_final_moderated_openai \n",
    "                    './results/<google_perspective_moderation_for_gpt_turbo_paraphrasing>.pkl', # comments_moderation_google_perspective_gpt-3.5_final\n",
    "                    './results/<google_palm_moderation_for_gpt_turbo_paraphrasing>.pkl', # comment_paraphrased_gpt-3.5_final_moderation_google_latest\n",
    "                    './results/<clarifai_moderation_for_gpt_turbo_paraphrasing>.pkl' # comment_paraphrased_gpt-3.5_final_moderation_clarifai\n",
    "                    ]\n",
    "                ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import utils\n",
    "\n",
    "def read_pickle(filepath):\n",
    "    with open(filepath, 'rb') as handle:\n",
    "        obj = pickle.load(handle)\n",
    "    return obj\n",
    "\n",
    "for moderation_array, perturbation in zip(gpt_moderations, perturbations):\n",
    "\n",
    "    moderation_map_openai = utils.process_openai(read_pickle(moderation_array[0]))\n",
    "    moderation_map_perspective = utils.process_perspective(read_pickle(moderation_array[1]))\n",
    "    moderation_map_google = utils.process_google(read_pickle(moderation_array[2]))\n",
    "    moderation_map_clarifai = utils.process_clarifai(read_pickle(moderation_array[3]))\n",
    "\n",
    "    all_keys = set(key for d in [moderation_map_openai, moderation_map_perspective, moderation_map_google, moderation_map_clarifai] for key in d.keys())\n",
    "\n",
    "    combined_dict = {}\n",
    "\n",
    "    for key in all_keys:\n",
    "        combined_dict[key] = {\n",
    "            'openai': moderation_map_openai.get(key, 'NULL'),\n",
    "            'clarifai': moderation_map_clarifai.get(key, 'NULL'),\n",
    "            'perspective': moderation_map_perspective.get(key, 'NULL'),\n",
    "            'google': moderation_map_google.get(key, 'NULL')\n",
    "        }\n",
    "\n",
    "    with open(f'./results/moderation_results_fairness_perturbed_{perturbation}.pkl', 'wb') as handle:\n",
    "        pickle.dump(combined_dict, handle)"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}